[HUDI-532] Add java doc for the test classes of hudi test suite (#1901)
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java
index d0b32ee..eda776e 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java
@@ -49,6 +49,9 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+/**
+ * Base test class for IT Test helps to run command and generate data.
+ */
public abstract class ITTestBase {
public static final Logger LOG = LogManager.getLogger(ITTestBase.class);
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java
index f6d9073..ff41b44 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java
@@ -52,6 +52,9 @@
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+/**
+ * Unit test against DeltaWriterAdapter, by testing writing DFS files.
+ */
public class TestDFSHoodieTestSuiteWriterAdapter extends UtilitiesTestBase {
private FilebasedSchemaProvider schemaProvider;
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java
index 7419667..8e175c5 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java
@@ -48,6 +48,9 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for {@link DeltaInputWriter}.
+ */
public class TestFileDeltaInputWriter extends UtilitiesTestBase {
private FilebasedSchemaProvider schemaProvider;
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java
index f414c92..1e5ca68 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java
@@ -30,6 +30,9 @@
import org.apache.hudi.integ.testsuite.dag.WorkflowDag;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for the build process of {@link DagNode} and {@link WorkflowDag}.
+ */
public class TestWorkflowBuilder {
@Test
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java
index d1c2b76..c48d1b1 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java
@@ -23,6 +23,7 @@
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+
import org.apache.avro.Schema.Field;
import org.apache.avro.generic.GenericRecord;
import org.apache.hudi.integ.testsuite.utils.TestUtils;
@@ -34,6 +35,9 @@
import org.junit.jupiter.api.Test;
import scala.Tuple2;
+/**
+ * Test Cases for {@link UpdateConverter} APIs.
+ */
public class TestUpdateConverter {
private JavaSparkContext jsc;
@@ -49,11 +53,16 @@
jsc.stop();
}
+ /**
+ * Test {@link UpdateConverter} by generates random updates from existing records.
+ */
@Test
public void testGenerateUpdateRecordsFromInputRecords() throws Exception {
+ // 1. prepare input records
JavaRDD<GenericRecord> inputRDD = TestUtils.makeRDD(jsc, 10);
String schemaStr = inputRDD.take(1).get(0).getSchema().toString();
int minPayloadSize = 1000;
+
// 2. DFS converter reads existing records and generates random updates for the same row keys
UpdateConverter updateConverter = new UpdateConverter(schemaStr, minPayloadSize,
Arrays.asList("timestamp"), Arrays.asList("_row_key"));
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java
index a91ea01..44b0ccd 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java
@@ -29,15 +29,20 @@
import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
import org.apache.spark.api.java.JavaRDD;
+/**
+ * An implementation of {@link WorkflowDagGenerator}, that generates complex workflowDag.
+ */
public class ComplexDagGenerator implements WorkflowDagGenerator {
@Override
public WorkflowDag build() {
+ // root node
DagNode root = new InsertNode(Config.newBuilder()
.withNumRecordsToInsert(1000)
.withNumInsertPartitions(3)
.withRecordSize(1000).build());
+ // child node1
DagNode child1 = new UpsertNode(Config.newBuilder()
.withNumRecordsToUpdate(999)
.withNumRecordsToInsert(1000)
@@ -46,6 +51,7 @@
.withNumInsertPartitions(1)
.withRecordSize(10000).build());
+ // function used to build ValidateNode
Function<List<DagNode<JavaRDD<WriteStatus>>>, Boolean> function = (dagNodes) -> {
DagNode<JavaRDD<WriteStatus>> parent1 = dagNodes.get(0);
List<WriteStatus> statuses = parent1.getResult().collect();
@@ -63,8 +69,11 @@
* parent2.getConfig().getNumInsertPartitions() + parent2.getConfig().getNumRecordsUpsert();
return b1 & b2 & b3;
};
+
+ // child node2
DagNode child2 = new ValidateNode(Config.newBuilder().build(), function);
+ // create relationship between nodes
root.addChildNode(child1);
// child1.addParentNode(root);
child1.addChildNode(child2);
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java
index 7309d43..e2f6ee5 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java
@@ -28,6 +28,9 @@
import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
import org.apache.hudi.integ.testsuite.dag.nodes.HiveQueryNode;
+/**
+ * An implementation of {@link WorkflowDagGenerator}, helps to generate a workflowDag with two hive nodes as child node.
+ */
public class HiveSyncDagGenerator implements WorkflowDagGenerator {
@Override
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java
index 26269fc..07dd467 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java
@@ -28,6 +28,10 @@
import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
import org.apache.hudi.integ.testsuite.dag.nodes.HiveQueryNode;
+/**
+ * An implementation of {@link WorkflowDagGenerator}, that generates workflowDag with hive nodes as child node
+ * for MOR table.
+ */
public class HiveSyncDagGeneratorMOR implements WorkflowDagGenerator {
@Override
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java
index b5a0cd3..418e644 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java
@@ -29,6 +29,9 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+/**
+ * A utility class for DAG test.
+ */
public class TestDagUtils {
private static final String COW_DAG_DOCKER_DEMO_RELATIVE_PATH = "/docker/demo/config/test-suite/complex-dag-cow.yaml";
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java
index 85d5358..c31a7d6 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java
@@ -25,6 +25,9 @@
import org.apache.hudi.utilities.testutils.UtilitiesTestBase;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for {@link GenericRecordFullPayloadSizeEstimator}.
+ */
public class TestGenericRecordPayloadEstimator {
private static final String SOURCE_SCHEMA_DOCKER_DEMO_RELATIVE_PATH = "/docker/demo/config/test-suite/source.avsc";
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java
index 886fb16..7524d4a 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java
@@ -32,6 +32,9 @@
import org.apache.hudi.utilities.testutils.UtilitiesTestBase;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for {@link GenericRecordFullPayloadGenerator} and {@link GenericRecordPartialPayloadGenerator}.
+ */
public class TestGenericRecordPayloadGenerator {
private static final String SOURCE_SCHEMA_DOCKER_DEMO_RELATIVE_PATH = "/docker/demo/config/test-suite/source.avsc";
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
index 4c71b60..5826019 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
@@ -49,6 +49,9 @@
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
+/**
+ * Unit test against {@link HoodieTestSuiteJob}.
+ */
public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
private static final String TEST_NAME_WITH_PARAMS = "[{index}] Test with useDeltaStreamer={0}, tableType={1}";
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java
index 6d0b994..fa8f4ac 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java
@@ -36,6 +36,9 @@
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for {@link DFSAvroDeltaInputReader} by issuing analyzeSingleFile and read from it.
+ */
public class TestDFSAvroDeltaInputReader extends UtilitiesTestBase {
@BeforeAll
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java
index 911f168..941ef43 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java
@@ -40,6 +40,9 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for {@link DFSHoodieDatasetInputReader}.
+ */
public class TestDFSHoodieDatasetInputReader extends UtilitiesTestBase {
@BeforeAll
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java
index 4699324..372d674 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java
@@ -28,6 +28,9 @@
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;
+/**
+ * A utility class for testing purpose.
+ */
public class TestUtils {
/**
@@ -45,6 +48,15 @@
return dataGenerator.generateGenericRecords(numRecords);
}
+ /**
+ * Method helps to create avro files and save it to file.
+ *
+ * @param jsc Java Spark Context jsc.
+ * @param sparkSession Spark Session, the entry point to programming Spark with the Dataset and DataFrame API.
+ * @param basePath The basePath where files are written.
+ * @param numFiles The number of files to create.
+ * @param numRecordsPerFile The number of records per file.
+ */
public static void createAvroFiles(JavaSparkContext jsc, SparkSession sparkSession, String basePath, int numFiles,
int numRecordsPerFile) {
Schema schema = HoodieTestDataGenerator.AVRO_SCHEMA;
