Merge branch 'cassandra-3.0' into cassandra-3.11
# Conflicts:
# CHANGES.txt
# src/java/org/apache/cassandra/db/ColumnFamilyStore.java
diff --git a/CHANGES.txt b/CHANGES.txt
index d4ce6dc..9463403 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -32,6 +32,7 @@
* Duplicate results with DISTINCT queries in mixed mode (CASSANDRA-15501)
* Disable JMX rebinding (CASSANDRA-15653)
Merged from 2.1:
+ * Fix writing of snapshot manifest when the table has table-backed secondary indexes (CASSANDRA-10968)
* Fix parse error in cqlsh COPY FROM and formatting for map of blobs (CASSANDRA-15679)
* Fix Commit log replays when static column clustering keys are collections (CASSANDRA-14365)
* Fix Red Hat init script on newer systemd versions (CASSANDRA-15273)
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index fa00e5b..99cac7c 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1797,9 +1797,9 @@
public Set<SSTableReader> snapshotWithoutFlush(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral)
{
Set<SSTableReader> snapshottedSSTables = new HashSet<>();
+ final JSONArray filesJSONArr = new JSONArray();
for (ColumnFamilyStore cfs : concatWithIndexes())
{
- final JSONArray filesJSONArr = new JSONArray();
try (RefViewFragment currentView = cfs.selectAndReference(View.select(SSTableSet.CANONICAL, (x) -> predicate == null || predicate.apply(x))))
{
for (SSTableReader ssTable : currentView.sstables)
@@ -1812,12 +1812,13 @@
logger.trace("Snapshot for {} keyspace data file {} created in {}", keyspace, ssTable.getFilename(), snapshotDirectory);
snapshottedSSTables.add(ssTable);
}
-
- writeSnapshotManifest(filesJSONArr, snapshotName);
- if (!SchemaConstants.isLocalSystemKeyspace(metadata.ksName) && !SchemaConstants.isReplicatedSystemKeyspace(metadata.ksName))
- writeSnapshotSchema(snapshotName);
}
}
+
+ writeSnapshotManifest(filesJSONArr, snapshotName);
+ if (!SchemaConstants.isLocalSystemKeyspace(metadata.ksName) && !SchemaConstants.isReplicatedSystemKeyspace(metadata.ksName))
+ writeSnapshotSchema(snapshotName);
+
if (ephemeral)
createEphemeralSnapshotMarkerFile(snapshotName);
return snapshottedSSTables;
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 0ffa6be..8665c8d 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -507,7 +507,7 @@
*/
public static File getSnapshotDirectory(File location, String snapshotName)
{
- if (location.getName().startsWith(SECONDARY_INDEX_NAME_SEPARATOR))
+ if (isSecondaryIndexFolder(location))
{
return getOrCreate(location.getParentFile(), SNAPSHOT_SUBDIR, snapshotName, location.getName());
}
@@ -547,7 +547,7 @@
public static File getBackupsDirectory(File location)
{
- if (location.getName().startsWith(SECONDARY_INDEX_NAME_SEPARATOR))
+ if (isSecondaryIndexFolder(location))
{
return getOrCreate(location.getParentFile(), BACKUPS_SUBDIR, location.getName());
}
@@ -851,9 +851,9 @@
final List<File> snapshots = new LinkedList<>();
for (final File dir : dataPaths)
{
- File snapshotDir = dir.getName().startsWith(SECONDARY_INDEX_NAME_SEPARATOR) ?
- new File(dir.getParent(), SNAPSHOT_SUBDIR) :
- new File(dir, SNAPSHOT_SUBDIR);
+ File snapshotDir = isSecondaryIndexFolder(dir)
+ ? new File(dir.getParent(), SNAPSHOT_SUBDIR)
+ : new File(dir, SNAPSHOT_SUBDIR);
if (snapshotDir.exists() && snapshotDir.isDirectory())
{
final File[] snapshotDirs = snapshotDir.listFiles();
@@ -876,7 +876,7 @@
for (File dir : dataPaths)
{
File snapshotDir;
- if (dir.getName().startsWith(SECONDARY_INDEX_NAME_SEPARATOR))
+ if (isSecondaryIndexFolder(dir))
{
snapshotDir = new File(dir.getParentFile(), join(SNAPSHOT_SUBDIR, snapshotName, dir.getName()));
}
@@ -935,9 +935,9 @@
long result = 0L;
for (File dir : dataPaths)
{
- File snapshotDir = dir.getName().startsWith(SECONDARY_INDEX_NAME_SEPARATOR) ?
- new File(dir.getParent(), SNAPSHOT_SUBDIR) :
- new File(dir, SNAPSHOT_SUBDIR);
+ File snapshotDir = isSecondaryIndexFolder(dir)
+ ? new File(dir.getParent(), SNAPSHOT_SUBDIR)
+ : new File(dir, SNAPSHOT_SUBDIR);
result += getTrueAllocatedSizeIn(snapshotDir);
}
return result;
@@ -999,6 +999,11 @@
return result;
}
+ public static boolean isSecondaryIndexFolder(File dir)
+ {
+ return dir.getName().startsWith(SECONDARY_INDEX_NAME_SEPARATOR);
+ }
+
public List<File> getCFDirectories()
{
List<File> result = new ArrayList<>();
diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
index 1f7e67f..b7155ef 100644
--- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java
+++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
@@ -153,6 +153,11 @@
public String relativeFilenameFor(Component component)
{
final StringBuilder buff = new StringBuilder();
+ if (Directories.isSecondaryIndexFolder(directory))
+ {
+ buff.append(directory.getName()).append(File.separator);
+ }
+
appendFileName(buff);
buff.append(separator).append(component.name());
return buff.toString();
@@ -290,7 +295,7 @@
File cfDirectory = parentDirectory;
// check if this is secondary index
String indexName = "";
- if (cfDirectory.getName().startsWith(Directories.SECONDARY_INDEX_NAME_SEPARATOR))
+ if (Directories.isSecondaryIndexFolder(cfDirectory))
{
indexName = cfDirectory.getName();
cfDirectory = cfDirectory.getParentFile();
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index 8463a1f..a3564bb 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -19,6 +19,7 @@
package org.apache.cassandra.db;
import java.io.File;
+import java.io.FileReader;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
@@ -29,6 +30,10 @@
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -52,6 +57,7 @@
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.WrappedRunnable;
import static junit.framework.Assert.assertNotNull;
+
@RunWith(OrderedJUnit4ClassRunner.class)
public class ColumnFamilyStoreTest
{
@@ -514,6 +520,39 @@
}
@Test
+ public void testSnapshotWithoutFlushWithSecondaryIndexes() throws Exception
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_INDEX1);
+ cfs.truncateBlocking();
+
+ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, "key")
+ .newRow()
+ .add("birthdate", 1L)
+ .add("notbirthdate", 2L);
+ new Mutation(builder.build()).applyUnsafe();
+ cfs.forceBlockingFlush();
+
+ String snapshotName = "newSnapshot";
+ cfs.snapshotWithoutFlush(snapshotName);
+
+ File snapshotManifestFile = cfs.getDirectories().getSnapshotManifestFile(snapshotName);
+ JSONParser parser = new JSONParser();
+ JSONObject manifest = (JSONObject) parser.parse(new FileReader(snapshotManifestFile));
+ JSONArray files = (JSONArray) manifest.get("files");
+
+ // Keyspace1-Indexed1 and the corresponding index
+ assert files.size() == 2;
+
+ // Snapshot of the secondary index is stored in the subfolder with the same file name
+ String baseTableFile = (String) files.get(0);
+ String indexTableFile = (String) files.get(1);
+ assert !baseTableFile.equals(indexTableFile);
+ assert Directories.isSecondaryIndexFolder(new File(indexTableFile).getParentFile());
+ assert indexTableFile.endsWith(baseTableFile);
+ }
+
+ @Test
public void testScrubDataDirectories() throws Throwable
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
diff --git a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
index 64367dc..9c1dc84 100644
--- a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
@@ -23,8 +23,10 @@
import org.apache.commons.lang3.StringUtils;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -45,6 +47,12 @@
tempDataDir = File.createTempFile("DescriptorTest", null).getParentFile();
}
+ @BeforeClass
+ public static void setup()
+ {
+ DatabaseDescriptor.daemonInitialization();
+ }
+
@Test
public void testFromFilename() throws Exception
{
