[HUDI-995] Migrate HoodieTestUtils APIs to HoodieTestTable (#2112)
Remove APIs in HoodieTestUtils
- HoodieTestUtils#createInflightCommitFiles
- HoodieTestUtils#getCommitFilePath
- HoodieTestUtils#doesCommitExist
and migrate usages to HoodieTestTable in
- hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
- hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
- hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java
- hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java
- hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java
index 7d5cee6..f69e3d3 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java
@@ -54,7 +54,7 @@
private String tablePath;
@BeforeEach
- public void init() throws IOException {
+ public void init() throws Exception {
initDFS();
jsc.hadoopConfiguration().addResource(dfs.getConf());
HoodieCLI.conf = dfs.getConf();
@@ -156,7 +156,7 @@
* Test for command: show archived commits.
*/
@Test
- public void testShowCommits() throws IOException {
+ public void testShowCommits() throws Exception {
CommandResult cr = getShell().executeCommand("show archived commits");
assertTrue(cr.isSuccess());
final List<Comparable[]> rows = new ArrayList<>();
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java
index 3da7189..2311aaa 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java
@@ -62,7 +62,7 @@
private URL propsFilePath;
@BeforeEach
- public void init() throws IOException {
+ public void init() throws Exception {
HoodieCLI.conf = jsc.hadoopConfiguration();
String tableName = "test_table";
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
index 44e2b80..c2509b1 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
@@ -76,17 +76,19 @@
"", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
}
- private LinkedHashMap<String, Integer[]> generateData() {
+ private LinkedHashMap<String, Integer[]> generateData() throws Exception {
// generate data and metadata
LinkedHashMap<String, Integer[]> data = new LinkedHashMap<>();
data.put("102", new Integer[] {15, 10});
data.put("101", new Integer[] {20, 10});
data.put("100", new Integer[] {15, 15});
- data.forEach((key, value) -> {
+ for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
+ String key = entry.getKey();
+ Integer[] value = entry.getValue();
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(),
Option.of(value[0]), Option.of(value[1]));
- });
+ }
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
assertEquals(3, metaClient.reloadActiveTimeline().getCommitsTimeline().countInstants(),
@@ -138,7 +140,7 @@
* Test case of 'commits show' command.
*/
@Test
- public void testShowCommits() throws IOException {
+ public void testShowCommits() throws Exception {
Map<String, Integer[]> data = generateData();
CommandResult cr = getShell().executeCommand("commits show");
@@ -154,7 +156,7 @@
* Test case of 'commits showarchived' command.
*/
@Test
- public void testShowArchivedCommits() throws IOException {
+ public void testShowArchivedCommits() throws Exception {
// Generate archive
HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(tablePath)
.withSchema(HoodieTestCommitMetadataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
@@ -168,10 +170,12 @@
data.put("102", new Integer[] {25, 45});
data.put("101", new Integer[] {35, 15});
- data.forEach((key, value) -> {
+ for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
+ String key = entry.getKey();
+ Integer[] value = entry.getValue();
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(),
Option.of(value[0]), Option.of(value[1]));
- });
+ }
// archive
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
@@ -198,7 +202,7 @@
* Test case of 'commit showpartitions' command.
*/
@Test
- public void testShowCommitPartitions() {
+ public void testShowCommitPartitions() throws Exception {
Map<String, Integer[]> data = generateData();
String commitInstant = "101";
@@ -235,7 +239,7 @@
* Test case of 'commit showfiles' command.
*/
@Test
- public void testShowCommitFiles() {
+ public void testShowCommitFiles() throws Exception {
Map<String, Integer[]> data = generateData();
String commitInstant = "101";
@@ -270,7 +274,7 @@
* Test case of 'commits compare' command.
*/
@Test
- public void testCompareCommits() throws IOException {
+ public void testCompareCommits() throws Exception {
Map<String, Integer[]> data = generateData();
String tableName2 = "test_table2";
@@ -278,10 +282,12 @@
HoodieTestUtils.init(jsc.hadoopConfiguration(), tablePath2, getTableType());
data.remove("102");
- data.forEach((key, value) -> {
+ for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
+ String key = entry.getKey();
+ Integer[] value = entry.getValue();
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath2, key, jsc.hadoopConfiguration(),
Option.of(value[0]), Option.of(value[1]));
- });
+ }
CommandResult cr = getShell().executeCommand(String.format("commits compare --path %s", tablePath2));
assertTrue(cr.isSuccess());
@@ -298,7 +304,7 @@
* Test case of 'commits sync' command.
*/
@Test
- public void testSyncCommits() throws IOException {
+ public void testSyncCommits() throws Exception {
Map<String, Integer[]> data = generateData();
String tableName2 = "test_table2";
@@ -306,10 +312,12 @@
HoodieTestUtils.init(jsc.hadoopConfiguration(), tablePath2, getTableType(), tableName2);
data.remove("102");
- data.forEach((key, value) -> {
+ for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
+ String key = entry.getKey();
+ Integer[] value = entry.getValue();
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath2, key, jsc.hadoopConfiguration(),
Option.of(value[0]), Option.of(value[1]));
- });
+ }
CommandResult cr = getShell().executeCommand(String.format("commits sync --path %s", tablePath2));
assertTrue(cr.isSuccess());
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
index e7a4603..10197a3 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
@@ -26,12 +26,12 @@
import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest;
import org.apache.hudi.client.HoodieWriteClient;
import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieWriteConfig;
@@ -41,14 +41,18 @@
import org.junit.jupiter.api.Test;
import org.springframework.shell.core.CommandResult;
-import java.io.File;
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.stream.Stream;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -59,39 +63,37 @@
public class TestRollbacksCommand extends AbstractShellIntegrationTest {
@BeforeEach
- public void init() throws IOException {
+ public void init() throws Exception {
String tableName = "test_table";
- String tablePath = basePath + File.separator + tableName;
+ String tablePath = Paths.get(basePath, tableName).toString();
new TableCommand().createTable(
tablePath, tableName, HoodieTableType.MERGE_ON_READ.name(),
"", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
-
+ metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
//Create some commits files and parquet files
- String commitTime1 = "100";
- String commitTime2 = "101";
- String commitTime3 = "102";
- HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
-
- // two commit files
- HoodieTestUtils.createCommitFiles(tablePath, commitTime1, commitTime2);
- // one .inflight commit file
- HoodieTestUtils.createInflightCommitFiles(tablePath, commitTime3);
-
- // generate commit files for commits
- for (String commitTime : Arrays.asList(commitTime1, commitTime2, commitTime3)) {
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
- }
-
+ Map<String, String> partitionAndFileId = new HashMap<String, String>() {
+ {
+ put(DEFAULT_FIRST_PARTITION_PATH, "file-1");
+ put(DEFAULT_SECOND_PARTITION_PATH, "file-2");
+ put(DEFAULT_THIRD_PARTITION_PATH, "file-3");
+ }
+ };
+ HoodieTestTable.of(metaClient)
+ .withPartitionMetaFiles(DEFAULT_PARTITION_PATHS)
+ .addCommit("100")
+ .withBaseFilesInPartitions(partitionAndFileId)
+ .addCommit("101")
+ .withBaseFilesInPartitions(partitionAndFileId)
+ .addInflightCommit("102")
+ .withBaseFilesInPartitions(partitionAndFileId);
// generate two rollback
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(tablePath)
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build();
try (HoodieWriteClient client = getHoodieWriteClient(config)) {
// Rollback inflight commit3 and commit2
- client.rollback(commitTime3);
- client.rollback(commitTime2);
+ client.rollback("102");
+ client.rollback("101");
}
}
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java
index 35e5b47..cd4f196 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java
@@ -73,17 +73,19 @@
* Test case for command 'stats wa'.
*/
@Test
- public void testWriteAmplificationStats() {
+ public void testWriteAmplificationStats() throws Exception {
// generate data and metadata
Map<String, Integer[]> data = new LinkedHashMap<>();
data.put("100", new Integer[] {15, 10});
data.put("101", new Integer[] {20, 10});
data.put("102", new Integer[] {15, 15});
- data.forEach((key, value) -> {
- HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(),
- Option.of(value[0]), Option.of(value[1]));
- });
+ for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
+ String k = entry.getKey();
+ Integer[] v = entry.getValue();
+ HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, k, jsc.hadoopConfiguration(),
+ Option.of(v[0]), Option.of(v[1]));
+ }
CommandResult cr = getShell().executeCommand("stats wa");
assertTrue(cr.isSuccess());
@@ -93,7 +95,7 @@
DecimalFormat df = new DecimalFormat("#.00");
data.forEach((key, value) -> {
// there are two partitions, so need to *2
- rows.add(new Comparable[]{key, value[1] * 2, value[0] * 2, df.format((float) value[0] / value[1])});
+ rows.add(new Comparable[] {key, value[1] * 2, value[0] * 2, df.format((float) value[0] / value[1])});
});
int totalWrite = data.values().stream().map(integers -> integers[0] * 2).mapToInt(s -> s).sum();
int totalUpdate = data.values().stream().map(integers -> integers[1] * 2).mapToInt(s -> s).sum();
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
index 2334f12..b9aa3f7 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
@@ -27,8 +27,7 @@
import org.apache.hudi.common.table.HoodieTableVersion;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.testutils.FileCreateUtils;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -36,11 +35,14 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import java.io.File;
import java.io.IOException;
-import java.util.Arrays;
+import java.nio.file.Paths;
import java.util.Properties;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
@@ -51,45 +53,31 @@
private String tablePath;
@BeforeEach
- public void init() throws IOException {
+ public void init() throws Exception {
String tableName = "test_table";
- tablePath = basePath + File.separator + tableName;
+ tablePath = Paths.get(basePath, tableName).toString();
new TableCommand().createTable(
tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(),
"", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
-
+ metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
//Create some commits files and parquet files
- String commitTime1 = "100";
- String commitTime2 = "101";
- HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
-
- // one commit file
- HoodieTestUtils.createCommitFiles(tablePath, commitTime1);
- // one .inflight commit file
- HoodieTestUtils.createInflightCommitFiles(tablePath, commitTime2);
-
- // generate commit files for commit 100
- for (String commitTime : Arrays.asList(commitTime1)) {
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
- }
-
- // generate commit and marker files for inflight commit 101
- for (String commitTime : Arrays.asList(commitTime2)) {
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
- FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1", IOType.MERGE);
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
- FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2", IOType.MERGE);
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
- FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3", IOType.MERGE);
- }
+ HoodieTestTable.of(metaClient)
+ .withPartitionMetaFiles(DEFAULT_PARTITION_PATHS)
+ .addCommit("100")
+ .withBaseFilesInPartition(DEFAULT_FIRST_PARTITION_PATH, "file-1")
+ .withBaseFilesInPartition(DEFAULT_SECOND_PARTITION_PATH, "file-2")
+ .withBaseFilesInPartition(DEFAULT_THIRD_PARTITION_PATH, "file-3")
+ .addInflightCommit("101")
+ .withBaseFilesInPartition(DEFAULT_FIRST_PARTITION_PATH, "file-1")
+ .withBaseFilesInPartition(DEFAULT_SECOND_PARTITION_PATH, "file-2")
+ .withBaseFilesInPartition(DEFAULT_THIRD_PARTITION_PATH, "file-3")
+ .withMarkerFile(DEFAULT_FIRST_PARTITION_PATH, "file-1", IOType.MERGE)
+ .withMarkerFile(DEFAULT_SECOND_PARTITION_PATH, "file-2", IOType.MERGE)
+ .withMarkerFile(DEFAULT_THIRD_PARTITION_PATH, "file-3", IOType.MERGE);
}
@Test
public void testDowngradeCommand() throws Exception {
- metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
-
// update hoodie.table.version to 1
metaClient.getTableConfig().setTableVersion(HoodieTableVersion.ONE);
try (FSDataOutputStream os = metaClient.getFs().create(new Path(metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE), true)) {
@@ -98,7 +86,7 @@
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
// verify marker files for inflight commit exists
- for (String partitionPath : HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS) {
+ for (String partitionPath : DEFAULT_PARTITION_PATHS) {
assertEquals(1, FileCreateUtils.getTotalMarkerFileCount(tablePath, partitionPath, "101", IOType.MERGE));
}
@@ -112,7 +100,7 @@
assertTableVersionFromPropertyFile();
// verify marker files are non existant
- for (String partitionPath : HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS) {
+ for (String partitionPath : DEFAULT_PARTITION_PATHS) {
assertEquals(0, FileCreateUtils.getTotalMarkerFileCount(tablePath, partitionPath, "101", IOType.MERGE));
}
}
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java
index aa87cdd..adc6145 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java
@@ -26,17 +26,21 @@
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.testutils.HoodieTestTable;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.shell.core.CommandResult;
-import java.io.File;
import java.io.IOException;
-import java.util.Arrays;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -48,43 +52,45 @@
*/
public class ITTestCommitsCommand extends AbstractShellIntegrationTest {
- private String tablePath;
-
@BeforeEach
public void init() throws IOException {
- String tableName = "test_table";
- tablePath = basePath + File.separator + tableName;
+ String tableName = "test_table_" + ITTestCommitsCommand.class.getName();
+ String tablePath = Paths.get(basePath, tableName).toString();
HoodieCLI.conf = jsc.hadoopConfiguration();
// Create table and connect
new TableCommand().createTable(
tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(),
"", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
+ metaClient.setBasePath(tablePath);
+ metaClient = HoodieTableMetaClient.reload(metaClient);
}
/**
* Test case of 'commit rollback' command.
*/
@Test
- public void testRollbackCommit() throws IOException {
+ public void testRollbackCommit() throws Exception {
//Create some commits files and parquet files
- String commitTime1 = "100";
- String commitTime2 = "101";
- String commitTime3 = "102";
- HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
-
- // three commit files
- HoodieTestUtils.createCommitFiles(tablePath, commitTime1, commitTime2, commitTime3);
-
- // generate commit files for commits
- for (String commitTime : Arrays.asList(commitTime1, commitTime2, commitTime3)) {
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
- HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
- }
+ Map<String, String> partitionAndFileId = new HashMap<String, String>() {
+ {
+ put(DEFAULT_FIRST_PARTITION_PATH, "file-1");
+ put(DEFAULT_SECOND_PARTITION_PATH, "file-2");
+ put(DEFAULT_THIRD_PARTITION_PATH, "file-3");
+ }
+ };
+ final String rollbackCommit = "102";
+ HoodieTestTable.of(metaClient)
+ .withPartitionMetaFiles(DEFAULT_PARTITION_PATHS)
+ .addCommit("100")
+ .withBaseFilesInPartitions(partitionAndFileId)
+ .addCommit("101")
+ .withBaseFilesInPartitions(partitionAndFileId)
+ .addCommit(rollbackCommit)
+ .withBaseFilesInPartitions(partitionAndFileId);
CommandResult cr = getShell().executeCommand(String.format("commit rollback --commit %s --sparkMaster %s --sparkMemory %s",
- commitTime3, "local", "4G"));
+ rollbackCommit, "local", "4G"));
assertTrue(cr.isSuccess());
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java
index b86e748..f4d8019 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java
@@ -23,18 +23,14 @@
import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.testutils.FileCreateUtils;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
-import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.exception.HoodieIOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
@@ -42,6 +38,9 @@
import java.util.Map;
import java.util.UUID;
+import static org.apache.hudi.common.testutils.FileCreateUtils.baseFileName;
+import static org.apache.hudi.common.util.CollectionUtils.createImmutableList;
+
/**
* Class to be used in tests to keep generating test inserts and updates against a corpus.
*/
@@ -62,67 +61,53 @@
/**
* Create a commit file with default CommitMetadata.
*/
- public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration) {
+ public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration) throws Exception {
createCommitFileWithMetadata(basePath, commitTime, configuration, Option.empty(), Option.empty());
}
public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration,
- Option<Integer> writes, Option<Integer> updates) {
+ Option<Integer> writes, Option<Integer> updates) throws Exception {
createCommitFileWithMetadata(basePath, commitTime, configuration, UUID.randomUUID().toString(),
UUID.randomUUID().toString(), writes, updates);
}
public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration,
- String fileId1, String fileId2, Option<Integer> writes, Option<Integer> updates) {
- Arrays.asList(HoodieTimeline.makeCommitFileName(commitTime), HoodieTimeline.makeInflightCommitFileName(commitTime),
- HoodieTimeline.makeRequestedCommitFileName(commitTime))
- .forEach(f -> {
- Path commitFile = new Path(
- basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + f);
- FSDataOutputStream os = null;
- try {
- FileSystem fs = FSUtils.getFs(basePath, configuration);
- os = fs.create(commitFile, true);
- // Generate commitMetadata
- HoodieCommitMetadata commitMetadata =
- generateCommitMetadata(basePath, commitTime, fileId1, fileId2, writes, updates);
- // Write empty commit metadata
- os.writeBytes(new String(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
- } catch (IOException ioe) {
- throw new HoodieIOException(ioe.getMessage(), ioe);
- } finally {
- if (null != os) {
- try {
- os.close();
- } catch (IOException e) {
- throw new HoodieIOException(e.getMessage(), e);
- }
- }
- }
- });
+ String fileId1, String fileId2, Option<Integer> writes, Option<Integer> updates) throws Exception {
+ List<String> commitFileNames = Arrays.asList(HoodieTimeline.makeCommitFileName(commitTime), HoodieTimeline.makeInflightCommitFileName(commitTime),
+ HoodieTimeline.makeRequestedCommitFileName(commitTime));
+ for (String name : commitFileNames) {
+ Path commitFilePath = new Path(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + name);
+ try (FSDataOutputStream os = FSUtils.getFs(basePath, configuration).create(commitFilePath, true)) {
+ // Generate commitMetadata
+ HoodieCommitMetadata commitMetadata =
+ generateCommitMetadata(basePath, commitTime, fileId1, fileId2, writes, updates);
+ // Write empty commit metadata
+ os.writeBytes(new String(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
+ }
+ }
}
/**
* Generate commitMetadata in path.
*/
- public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime) throws IOException {
+ public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime) throws Exception {
return generateCommitMetadata(basePath, commitTime, Option.empty(), Option.empty());
}
public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime,
- Option<Integer> writes, Option<Integer> updates) throws IOException {
+ Option<Integer> writes, Option<Integer> updates) throws Exception {
return generateCommitMetadata(basePath, commitTime, UUID.randomUUID().toString(), UUID.randomUUID().toString(),
writes, updates);
}
public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime, String fileId1,
- String fileId2, Option<Integer> writes, Option<Integer> updates) throws IOException {
- String file1P0C0 = HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, commitTime, fileId1);
- String file1P1C0 = HoodieTestUtils.createDataFile(basePath, DEFAULT_SECOND_PARTITION_PATH, commitTime, fileId2);
+ String fileId2, Option<Integer> writes, Option<Integer> updates) throws Exception {
+ FileCreateUtils.createBaseFile(basePath, DEFAULT_FIRST_PARTITION_PATH, commitTime, fileId1);
+ FileCreateUtils.createBaseFile(basePath, DEFAULT_SECOND_PARTITION_PATH, commitTime, fileId2);
return generateCommitMetadata(new HashMap<String, List<String>>() {
{
- put(DEFAULT_FIRST_PARTITION_PATH, CollectionUtils.createImmutableList(file1P0C0));
- put(DEFAULT_SECOND_PARTITION_PATH, CollectionUtils.createImmutableList(file1P1C0));
+ put(DEFAULT_FIRST_PARTITION_PATH, createImmutableList(baseFileName(DEFAULT_FIRST_PARTITION_PATH, fileId1)));
+ put(DEFAULT_SECOND_PARTITION_PATH, createImmutableList(baseFileName(DEFAULT_SECOND_PARTITION_PATH, fileId2)));
}
}, writes, updates);
}
diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
index ba4ffb4..c9a03c7 100644
--- a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
@@ -33,7 +33,7 @@
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.testutils.RawTripTestPayload;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.Option;
@@ -53,20 +53,22 @@
import org.apache.hudi.table.action.commit.WriteHelper;
import org.apache.hudi.testutils.HoodieClientTestBase;
import org.apache.hudi.testutils.HoodieClientTestUtils;
+import org.apache.hudi.testutils.HoodieWriteableTestTable;
import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.ValueSource;
-import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -102,6 +104,12 @@
public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
private static final Logger LOG = LogManager.getLogger(TestHoodieClientOnCopyOnWriteStorage.class);
+ private HoodieTestTable testTable;
+
+ @BeforeEach
+ public void setUpTestTable() {
+ testTable = HoodieWriteableTestTable.of(metaClient);
+ }
/**
* Test Auto Commit behavior for HoodieWriteClient insert API.
@@ -170,10 +178,10 @@
JavaRDD<WriteStatus> result = insertFirstBatch(cfg, client, newCommitTime, prevCommitTime, numRecords, writeFn,
isPrepped, false, numRecords);
- assertFalse(HoodieTestUtils.doesCommitExist(basePath, newCommitTime),
+ assertFalse(testTable.commitExists(newCommitTime),
"If Autocommit is false, then commit should not be made automatically");
assertTrue(client.commit(newCommitTime, result), "Commit should succeed");
- assertTrue(HoodieTestUtils.doesCommitExist(basePath, newCommitTime),
+ assertTrue(testTable.commitExists(newCommitTime),
"After explicit commit, commit file should be created");
}
}
@@ -985,7 +993,7 @@
JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, instantTime);
assertTrue(client.commit(instantTime, result), "Commit should succeed");
- assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
+ assertTrue(testTable.commitExists(instantTime),
"After explicit commit, commit file should be created");
// Get parquet file paths from commit metadata
@@ -998,16 +1006,14 @@
Collection<String> commitPathNames = commitMetadata.getFileIdAndFullPaths(basePath).values();
// Read from commit file
- String filename = HoodieTestUtils.getCommitFilePath(basePath, instantTime);
- FileInputStream inputStream = new FileInputStream(filename);
- String everything = FileIOUtils.readAsUTFString(inputStream);
- HoodieCommitMetadata metadata = HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class);
- HashMap<String, String> paths = metadata.getFileIdAndFullPaths(basePath);
- inputStream.close();
-
- // Compare values in both to make sure they are equal.
- for (String pathName : paths.values()) {
- assertTrue(commitPathNames.contains(pathName));
+ try (FSDataInputStream inputStream = fs.open(testTable.getCommitFilePath(instantTime))) {
+ String everything = FileIOUtils.readAsUTFString(inputStream);
+ HoodieCommitMetadata metadata = HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class);
+ HashMap<String, String> paths = metadata.getFileIdAndFullPaths(basePath);
+ // Compare values in both to make sure they are equal.
+ for (String pathName : paths.values()) {
+ assertTrue(commitPathNames.contains(pathName));
+ }
}
}
}
@@ -1017,65 +1023,61 @@
*/
@Test
public void testMetadataStatsOnCommit() throws Exception {
-
HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).build();
HoodieWriteClient client = getHoodieWriteClient(cfg);
- HoodieTableMetaClient metaClient = new HoodieTableMetaClient(hadoopConf, basePath);
- String instantTime = "000";
- client.startCommitWithTime(instantTime);
+ String instantTime0 = "000";
+ client.startCommitWithTime(instantTime0);
- List<HoodieRecord> records = dataGen.generateInserts(instantTime, 200);
- JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1);
+ List<HoodieRecord> records0 = dataGen.generateInserts(instantTime0, 200);
+ JavaRDD<HoodieRecord> writeRecords0 = jsc.parallelize(records0, 1);
+ JavaRDD<WriteStatus> result0 = client.bulkInsert(writeRecords0, instantTime0);
- JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, instantTime);
-
- assertTrue(client.commit(instantTime, result), "Commit should succeed");
- assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
+ assertTrue(client.commit(instantTime0, result0), "Commit should succeed");
+ assertTrue(testTable.commitExists(instantTime0),
"After explicit commit, commit file should be created");
// Read from commit file
- String filename = HoodieTestUtils.getCommitFilePath(basePath, instantTime);
- FileInputStream inputStream = new FileInputStream(filename);
- String everything = FileIOUtils.readAsUTFString(inputStream);
- HoodieCommitMetadata metadata =
- HoodieCommitMetadata.fromJsonString(everything.toString(), HoodieCommitMetadata.class);
- int inserts = 0;
- for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
- for (HoodieWriteStat stat : pstat.getValue()) {
- inserts += stat.getNumInserts();
+ try (FSDataInputStream inputStream = fs.open(testTable.getCommitFilePath(instantTime0))) {
+ String everything = FileIOUtils.readAsUTFString(inputStream);
+ HoodieCommitMetadata metadata =
+ HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class);
+ int inserts = 0;
+ for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
+ for (HoodieWriteStat stat : pstat.getValue()) {
+ inserts += stat.getNumInserts();
+ }
}
+ assertEquals(200, inserts);
}
- assertEquals(200, inserts);
// Update + Inserts such that they just expand file1
- instantTime = "001";
- client.startCommitWithTime(instantTime);
+ String instantTime1 = "001";
+ client.startCommitWithTime(instantTime1);
- records = dataGen.generateUpdates(instantTime, records);
- writeRecords = jsc.parallelize(records, 1);
- result = client.upsert(writeRecords, instantTime);
+ List<HoodieRecord> records1 = dataGen.generateUpdates(instantTime1, records0);
+ JavaRDD<HoodieRecord> writeRecords1 = jsc.parallelize(records1, 1);
+ JavaRDD<WriteStatus> result1 = client.upsert(writeRecords1, instantTime1);
- assertTrue(client.commit(instantTime, result), "Commit should succeed");
- assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
+ assertTrue(client.commit(instantTime1, result1), "Commit should succeed");
+ assertTrue(testTable.commitExists(instantTime1),
"After explicit commit, commit file should be created");
// Read from commit file
- filename = HoodieTestUtils.getCommitFilePath(basePath, instantTime);
- inputStream = new FileInputStream(filename);
- everything = FileIOUtils.readAsUTFString(inputStream);
- metadata = HoodieCommitMetadata.fromJsonString(everything.toString(), HoodieCommitMetadata.class);
- inserts = 0;
- int upserts = 0;
- for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
- for (HoodieWriteStat stat : pstat.getValue()) {
- inserts += stat.getNumInserts();
- upserts += stat.getNumUpdateWrites();
+ try (FSDataInputStream inputStream = fs.open(testTable.getCommitFilePath(instantTime1))) {
+ String everything = FileIOUtils.readAsUTFString(inputStream);
+ HoodieCommitMetadata metadata = HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class);
+ int inserts = 0;
+ int upserts = 0;
+ for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
+ for (HoodieWriteStat stat : pstat.getValue()) {
+ inserts += stat.getNumInserts();
+ upserts += stat.getNumUpdateWrites();
+ }
}
+ assertEquals(0, inserts);
+ assertEquals(200, upserts);
}
- assertEquals(0, inserts);
- assertEquals(200, upserts);
-
}
/**
@@ -1095,13 +1097,13 @@
metaClient.getFs().delete(result.getKey(), false);
if (!enableOptimisticConsistencyGuard) {
assertTrue(client.commit(instantTime, result.getRight()), "Commit should succeed");
- assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
+ assertTrue(testTable.commitExists(instantTime),
"After explicit commit, commit file should be created");
// Marker directory must be removed
assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
} else {
// with optimistic, first client.commit should have succeeded.
- assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
+ assertTrue(testTable.commitExists(instantTime),
"After explicit commit, commit file should be created");
// Marker directory must be removed
assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
@@ -1124,13 +1126,13 @@
if (!enableOptimisticConsistencyGuard) {
// Rollback of this commit should succeed with FailSafeCG
client.rollback(instantTime);
- assertFalse(HoodieTestUtils.doesCommitExist(basePath, instantTime),
+ assertFalse(testTable.commitExists(instantTime),
"After explicit rollback, commit file should not be present");
// Marker directory must be removed after rollback
assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
} else {
// if optimistic CG is enabled, commit should have succeeded.
- assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
+ assertTrue(testTable.commitExists(instantTime),
"With optimistic CG, first commit should succeed. commit file should be present");
// Marker directory must be removed after rollback
assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
@@ -1145,7 +1147,7 @@
} else {
// rollback of a completed commit should succeed if using list based rollback
client.rollback(instantTime);
- assertFalse(HoodieTestUtils.doesCommitExist(basePath, instantTime),
+ assertFalse(testTable.commitExists(instantTime),
"After explicit rollback, commit file should not be present");
}
}
@@ -1216,7 +1218,7 @@
JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(dataGen.generateInserts(firstInstantTime, numRecords), 1);
JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, firstInstantTime);
assertTrue(client.commit(firstInstantTime, result), "Commit should succeed");
- assertTrue(HoodieTestUtils.doesCommitExist(basePath, firstInstantTime),
+ assertTrue(testTable.commitExists(firstInstantTime),
"After explicit commit, commit file should be created");
// Check the entire dataset has all records still
@@ -1235,7 +1237,7 @@
JavaRDD<WriteStatus> inserts = client.bulkInsert(insertRecords, nextInstantTime);
JavaRDD<WriteStatus> upserts = client.upsert(updateRecords, nextInstantTime);
assertTrue(client.commit(nextInstantTime, inserts.union(upserts)), "Commit should succeed");
- assertTrue(HoodieTestUtils.doesCommitExist(basePath, firstInstantTime),
+ assertTrue(testTable.commitExists(firstInstantTime),
"After explicit commit, commit file should be created");
int totalRecords = 2 * numRecords;
assertEquals(totalRecords, HoodieClientTestUtils.read(jsc, basePath, sqlContext, fs, fullPartitionPaths).count(),
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
index 7f00f5a..a1dbe08 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
@@ -172,19 +172,6 @@
}
}
- /**
- * @deprecated Use {@link HoodieTestTable} instead.
- */
- public static void createInflightCommitFiles(String basePath, String... instantTimes) throws IOException {
-
- for (String instantTime : instantTimes) {
- new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
- + HoodieTimeline.makeRequestedCommitFileName(instantTime)).createNewFile();
- new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeInflightCommitFileName(
- instantTime)).createNewFile();
- }
- }
-
public static void createPendingCleanFiles(HoodieTableMetaClient metaClient, String... instantTimes) {
for (String instantTime : instantTimes) {
Arrays.asList(HoodieTimeline.makeRequestedCleanerFileName(instantTime),
@@ -257,22 +244,6 @@
TimelineMetadataUtils.serializeCompactionPlan(plan));
}
- /**
- * @deprecated Use {@link HoodieTestTable} instead.
- */
- public static String getCommitFilePath(String basePath, String instantTime) {
- return basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + instantTime + HoodieTimeline.COMMIT_EXTENSION;
- }
-
- /**
- * @deprecated Use {@link HoodieTestTable} instead.
- */
- public static boolean doesCommitExist(String basePath, String instantTime) {
- return new File(
- basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + instantTime + HoodieTimeline.COMMIT_EXTENSION)
- .exists();
- }
-
public static void createCleanFiles(HoodieTableMetaClient metaClient, String basePath,
String instantTime, Configuration configuration)
throws IOException {
