Merge remote-tracking branch master into FALCON-585.
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index b74a4ee..2f5de05 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -9,6 +9,24 @@
via Samarth Gupta)
IMPROVEMENTS
+ FALCON-585 Remove requirement to have write access to / (Raghav Kumar Gautam)
+ FALCON-680: Tests using LocalDC, impressionRC, updateBundle should stop using root dir
+ (Raghav Kumar Gautam)
+
+ FALCON-679: Tests using ELbundle should stop writing to root dir (Raghav Kumar Gautam)
+
+ FALCON-641 Tests using LateDataBundles should stop using root dir (Raghav
+ Kumar Gautam via Arpit Gupta)
+
+ FALCON-633 RetryTests and Retentions tests should stop using root dir
+ (Raghav Kumar Gautam via Samarth Gupta)
+
+ FALCON-629 Add javadoc for method stitchTwoHdfsPaths (Raghav Kumar Gautam
+ via Samarth Gupta)
+
+ FALCON-606 hcat tests should stop using root dir (Raghav Kumar Gautam
+ via Arpit Gupta)
+
FALCON-675 Request URLS moved from parameters into methods in falcon-regression (Ruslan
Ostafiychuk)
@@ -41,7 +59,7 @@
(Paul Isaychuk via Samarth Gupta)
FALCON-619 ELExp_FutureAndLatestTest stabilization (Paul Isaychuk via Arpit Gupta)
-
+
FALCON-610 Refactoring and documentation updates (Paul Isaychuk via Arpit Gupta)
FALCON-581 Refactor code for cross product and make it a method
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
index 6d8b032..1536d2c 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java
@@ -44,6 +44,7 @@
import org.apache.falcon.regression.core.helpers.ColoHelper;
import org.apache.falcon.regression.core.response.ServiceResponse;
import org.apache.falcon.regression.core.util.AssertUtil;
+import org.apache.falcon.regression.core.util.HadoopUtil;
import org.apache.falcon.regression.core.util.TimeUtil;
import org.apache.falcon.regression.core.util.Util;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
@@ -1015,4 +1016,22 @@
return feedObject.getName();
}
+ /**
+ * Update locations in cluster and base them at test path. For eg. the staging location will be
+ * {testPath}/staging
+ * @param testAppPath test app path of which locations will be based
+ */
+ public void updateClusterLocations(String testAppPath) {
+ List<String> newClusters = new ArrayList<String>();
+ for (String cluster : clusters) {
+ ClusterMerlin clusterMerlin = new ClusterMerlin(cluster);
+ for (org.apache.falcon.entity.v0.cluster.Location location : clusterMerlin
+ .getLocations().getLocations()) {
+ location.setPath(HadoopUtil.stitchHdfsPath(testAppPath, location.getName()));
+ }
+ newClusters.add(clusterMerlin.toString());
+ }
+ clusters = newClusters;
+ }
+
}
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
index 922c030..62838f8 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/BundleUtil.java
@@ -44,38 +44,52 @@
}
private static final Logger LOGGER = Logger.getLogger(BundleUtil.class);
- public static Bundle readLateDataBundle() throws IOException {
- return readBundleFromFolder("LateDataBundles");
+ public static Bundle readLateDataBundle(String appPath, String testName) throws IOException {
+ return generateBundleFromTemplate("LateDataBundles", appPath, testName);
}
- public static Bundle readRetryBundle() throws IOException {
- return readBundleFromFolder("RetryTests");
+ public static Bundle readRetryBundle(String appPath, String testName) throws IOException {
+ return generateBundleFromTemplate("RetryTests", appPath, testName);
}
- public static Bundle readRetentionBundle() throws IOException {
- return readBundleFromFolder("RetentionBundles");
+ public static Bundle readRetentionBundle(String appPath, String testName) throws IOException {
+ return generateBundleFromTemplate("RetentionBundles", appPath, testName);
}
- public static Bundle readELBundle() throws IOException {
- return readBundleFromFolder("ELbundle");
+ public static Bundle readELBundle(String appPath, String testName) throws IOException {
+ return generateBundleFromTemplate("ELbundle", appPath, testName);
}
- public static Bundle readHCatBundle() throws IOException {
- return readBundleFromFolder("hcat");
+ public static Bundle readHCatBundle(String appPath, String testName) throws IOException {
+ return generateBundleFromTemplate("hcat", appPath, testName);
}
- public static Bundle readHCat2Bundle() throws IOException {
- return readBundleFromFolder("hcat_2");
+ public static Bundle readHCat2Bundle(String appPath, String testName) throws IOException {
+ return generateBundleFromTemplate("hcat_2", appPath, testName);
}
- public static Bundle readLocalDCBundle() throws IOException {
- return readBundleFromFolder("LocalDC_feedReplicaltion_BillingRC");
+ public static Bundle readLocalDCBundle(String appPath, String testName) throws IOException {
+ return generateBundleFromTemplate("LocalDC_feedReplicaltion_BillingRC", appPath, testName);
}
- public static Bundle readUpdateBundle() throws IOException {
- return readBundleFromFolder("updateBundle");
+ public static Bundle readUpdateBundle(String appPath, String testName) throws IOException {
+ return generateBundleFromTemplate("updateBundle", appPath, testName);
}
+ /** Generate a bundle from the template stored in a directory
+ * @param templatePath name of directory where template is stored
+ * @param appPath application path where staging/temp/working directory will live
+ * @param testName name of test
+ * @return the customized bundle
+ * @throws IOException
+ */
+ private static Bundle generateBundleFromTemplate(String templatePath, String appPath,
+ String testName) throws IOException {
+ final Bundle bundle = readBundleFromFolder(templatePath);
+ bundle.updateClusterLocations(HadoopUtil.stitchHdfsPath(appPath, testName));
+ return bundle;
+ }
+
private static Bundle readBundleFromFolder(final String folderPath) throws IOException {
LOGGER.info("Loading xmls from directory: " + folderPath);
File directory = null;
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
index af5c01a..735a732 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
@@ -368,4 +368,32 @@
copyDataToFolders(fs, folderPrefix, folderPaths,
OSUtil.NORMAL_INPUT + "_SUCCESS", OSUtil.NORMAL_INPUT + "log_01.txt");
}
+
+ /**
+ * Stitch two or more hadoop paths. For eg: stitchHdfsPath("/tmp/", "/test") = "/tmp/test",
+ * stitchHdfsPath("/tmp/", "test") = "/tmp/test", stitchHdfsPath("/tmp", "test") = "/tmp/test"
+ * @param path1 the first path to be stitched
+ * @param path2 the second path to be stitched
+ * @param pathParts other paths to be stitched
+ * @return final stitched path
+ */
+ public static String stitchHdfsPath(String path1, String path2, String... pathParts) {
+ String retValue = stitchTwoHdfsPaths(path1, path2);
+ for (String pathPart : pathParts) {
+ retValue = stitchTwoHdfsPaths(retValue, pathPart);
+ }
+ return retValue;
+ }
+
+ /**
+ * Stitch two hadoop paths. For eg: stitchTwoHdfsPaths("/tmp/", "/test") = "/tmp/test",
+ * stitchTwoHdfsPaths("/tmp/", "test") = "/tmp/test",
+ * stitchTwoHdfsPaths("/tmp", "test") = "/tmp/test"
+ * @param path1 the first path to be stitched
+ * @param path2 the second path to be stitched
+ * @return final stitched path
+ */
+ private static String stitchTwoHdfsPaths(String path1, String path2) {
+ return path1.replaceAll("/*$", "") + "/" + path2.replaceAll("^/*", "");
+ }
}
diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
index c16c8f4..bc78699 100644
--- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
+++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java
@@ -45,11 +45,14 @@
}
}
- public ColoHelper prism;
- public List<ColoHelper> servers;
- public List<FileSystem> serverFS;
- public List<OozieClient> serverOC;
- public String baseHDFSDir = "/tmp/falcon-regression";
+ public final ColoHelper prism;
+ public final List<ColoHelper> servers;
+ public final List<FileSystem> serverFS;
+ public final List<OozieClient> serverOC;
+ public final String baseHDFSDir = Config.getProperty("falcon.regression.test.dir",
+ "/tmp/falcon-regression");
+ public final String baseAppHDFSDir = Config.getProperty("falcon.regression.app.dir",
+ "/tmp/falcon-regression-app");
public static final String PRISM_PREFIX = "prism";
protected Bundle[] bundles;
public static final String MINUTE_DATE_PATTERN = "/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}";
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java
index 4636416..85944ef 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java
@@ -80,7 +80,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundle, cluster);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExp_FutureAndLatestTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExp_FutureAndLatestTest.java
index 25e2dfe..24e94ea 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExp_FutureAndLatestTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExp_FutureAndLatestTest.java
@@ -52,6 +52,8 @@
FileSystem clusterFS = serverFS.get(0);
OozieClient clusterOC = serverOC.get(0);
private String baseTestDir = baseHDFSDir + "/ELExp_FutureAndLatest";
+ private String testInputDir = baseTestDir + "/input";
+ private String testOutputDir = baseTestDir + "/output";
private String aggregateWorkflowDir = baseTestDir + "/aggregator";
private static final Logger logger = Logger.getLogger(ELExp_FutureAndLatestTest.class);
@@ -60,14 +62,14 @@
logger.info("in @BeforeClass");
uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle b = BundleUtil.readELBundle();
+ Bundle b = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
b.generateUniqueBundle();
b = new Bundle(b, cluster);
String startDate = TimeUtil.getTimeWrtSystemTime(-20);
String endDate = TimeUtil.getTimeWrtSystemTime(70);
- b.setInputFeedDataPath(baseTestDir + "/ELExp_latest/testData" + MINUTE_DATE_PATTERN);
+ b.setInputFeedDataPath(testInputDir + MINUTE_DATE_PATTERN);
b.setProcessWorkflow(aggregateWorkflowDir);
List<String> dataDates = TimeUtil.getMinuteDatesOnEitherSide(startDate, endDate, 1);
@@ -79,11 +81,11 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
- bundles[0].setInputFeedDataPath(baseTestDir + "/ELExp_latest/testData" +
- MINUTE_DATE_PATTERN);
+ bundles[0].setInputFeedDataPath(testInputDir + MINUTE_DATE_PATTERN);
+ bundles[0].setOutputFeedLocationData(testOutputDir + MINUTE_DATE_PATTERN);
bundles[0].setInputFeedPeriodicity(5, TimeUnit.minutes);
bundles[0].setInputFeedValidity("2010-04-01T00:00Z", "2015-04-01T00:00Z");
String processStart = TimeUtil.getTimeWrtSystemTime(-3);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java
index 7555c0c..b5d1c62 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java
@@ -130,7 +130,7 @@
throws IOException, JAXBException, ParseException, URISyntaxException {
HadoopUtil.uploadDir(server.getClusterHelper().getHadoopFS(),
aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundle = new Bundle(bundle, server.getPrefix());
bundle.generateUniqueBundle();
bundle.setProcessWorkflow(aggregateWorkflowDir);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java
index 99ffe37..85fed2f 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java
@@ -77,7 +77,7 @@
//copy pig script
HadoopUtil.uploadDir(clusterFS, pigScriptDir, OSUtil.RESOURCES + "pig");
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundle.generateUniqueBundle();
bundle = new Bundle(bundle, cluster);
String startDate = "2010-01-02T00:40Z";
@@ -91,7 +91,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setInputFeedDataPath(inputPath);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedClusterUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedClusterUpdateTest.java
index 3499a4b..5e7a3bb 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedClusterUpdateTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedClusterUpdateTest.java
@@ -67,7 +67,7 @@
@BeforeClass(alwaysRun = true)
public void createTestData() throws Exception {
uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
for (int i = 0; i < 3; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
@@ -89,7 +89,7 @@
public void setup(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
for (int i = 0; i < 3; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java
index 33a81d0..a0228d6 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java
@@ -68,7 +68,7 @@
@BeforeMethod(alwaysRun = true)
public void testName(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
for (int i = 0; i < 3; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
index 197c3c6..bd786f4 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java
@@ -81,7 +81,8 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws JAXBException, IOException {
LOGGER.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readLocalDCBundle();
+ Bundle bundle =
+ BundleUtil.readLocalDCBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundle, cluster1);
bundles[1] = new Bundle(bundle, cluster2);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java
index bb629b8..f7a706d 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java
@@ -54,7 +54,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0].generateUniqueBundle();
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].submitClusters(prism);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java
index 8500d84..8f0ee4e 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java
@@ -50,7 +50,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
Bundle.submitCluster(bundles[0]);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java
index 6278513..4ee229f 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java
@@ -54,7 +54,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0].generateUniqueBundle();
bundles[0] = new Bundle(bundles[0], cluster);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java
index db82a8c..2c246de 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java
@@ -57,7 +57,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
feed = bundles[0].getDataSets().get(0);
@@ -165,4 +165,4 @@
public void tearDownClass() throws IOException {
cleanTestDirs();
}
-}
\ No newline at end of file
+}
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java
index 8d96741..3bd71ab 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java
@@ -47,7 +47,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0].generateUniqueBundle();
bundles[0] = new Bundle(bundles[0], cluster);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java
index 9676a30..adab5a6 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java
@@ -50,7 +50,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0].generateUniqueBundle();
bundles[0] = new Bundle(bundles[0], cluster);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java
index c56310e..86645a7 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java
@@ -78,7 +78,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- processBundle = BundleUtil.readELBundle();
+ processBundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
processBundle = new Bundle(processBundle, cluster1);
processBundle.generateUniqueBundle();
processBundle.setInputFeedDataPath(feedInputPath);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java
index db600ff..6c873c3 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java
@@ -62,6 +62,7 @@
String baseTestHDFSDir = baseHDFSDir + "/InstanceSummaryTest";
String feedInputPath = baseTestHDFSDir + "/testInputData" + MINUTE_DATE_PATTERN;
+ String feedOutputPath = baseTestHDFSDir + "/testOutputData" + MINUTE_DATE_PATTERN;
String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator";
String startTime;
String endTime;
@@ -87,10 +88,11 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
logger.info("test name: " + method.getName());
- processBundle = BundleUtil.readELBundle();
+ processBundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
processBundle = new Bundle(processBundle, cluster3);
processBundle.generateUniqueBundle();
processBundle.setInputFeedDataPath(feedInputPath);
+ processBundle.setOutputFeedLocationData(feedOutputPath);
processBundle.setProcessWorkflow(aggregateWorkflowDir);
for (int i = 0; i < 3; i++) {
@@ -266,8 +268,7 @@
@AfterMethod(alwaysRun = true)
public void tearDown() throws IOException {
- processBundle.deleteBundle(prism);
- removeBundles();
+ removeBundles(processBundle);
}
@AfterClass(alwaysRun = true)
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
index d4f31e9..27d4fdf 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java
@@ -75,8 +75,10 @@
DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy/MM/dd/HH/mm");
final private String baseTestDir = baseHDFSDir + "/NewRetryTest";
final private String aggregateWorkflowDir = baseTestDir + "/aggregator";
- final private String lateDir = baseTestDir + "/lateDataTest/testFolders";
- final private String latePath = lateDir + MINUTE_DATE_PATTERN;
+ final private String lateInputDir = baseTestDir + "/lateDataTest/inputFolders/";
+ final private String lateInputPath = lateInputDir + MINUTE_DATE_PATTERN;
+ final private String lateOutputDir = baseTestDir + "/lateDataTest/outputFolders/";
+ final private String lateOutputPath = lateOutputDir + MINUTE_DATE_PATTERN;
private DateTime startDate;
private DateTime endDate;
@@ -87,15 +89,17 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
- bundles[0] = new Bundle(BundleUtil.readRetryBundle(), cluster);
+ bundles[0] = new Bundle(
+ BundleUtil.readRetryBundle(baseAppHDFSDir, this.getClass().getSimpleName()), cluster);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
startDate = new DateTime(DateTimeZone.UTC).plusMinutes(1);
endDate = new DateTime(DateTimeZone.UTC).plusMinutes(2);
bundles[0].setProcessValidity(startDate, endDate);
+ bundles[0].setOutputFeedLocationData(lateOutputPath);
String feed =
- Util.setFeedPathValue(bundles[0].getInputFeedFromBundle(), latePath);
+ Util.setFeedPathValue(bundles[0].getInputFeedFromBundle(), lateInputPath);
feed = Util.insertLateFeedValue(feed, new Frequency("minutes(8)"));
bundles[0].getDataSets().remove(bundles[0].getInputFeedFromBundle());
bundles[0].getDataSets().add(feed);
@@ -127,8 +131,8 @@
} else {
AssertUtil.assertSucceeded(response);
// lets create data now:
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
//schedule process
AssertUtil.assertSucceeded(
@@ -178,8 +182,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
//now wait till the process is over
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
@@ -236,8 +240,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
@@ -289,8 +293,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
@@ -345,8 +349,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
//now wait till the process is over
@@ -393,8 +397,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
//now wait till the process is over
@@ -444,8 +448,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
//now wait till the process is over
@@ -498,8 +502,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
//now wait till the process is over
@@ -550,8 +554,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
//now wait till the process is over
@@ -589,8 +593,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
@@ -643,8 +647,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
//now wait till the process is over
@@ -680,7 +684,7 @@
public void testRetryInSuspendedAndResumeCaseWithLateData(Retry retry) throws Exception {
String feed =
- Util.setFeedPathValue(bundles[0].getInputFeedFromBundle(), latePath);
+ Util.setFeedPathValue(bundles[0].getInputFeedFromBundle(), lateInputPath);
feed = Util.insertLateFeedValue(feed, new Frequency("minutes(10)"));
bundles[0].getDataSets().remove(bundles[0].getInputFeedFromBundle());
bundles[0].getDataSets().add(feed);
@@ -698,8 +702,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
String bundleId = OozieUtil.getBundles(clusterOC,
@@ -774,7 +778,7 @@
public void testRetryInLateDataCase(Retry retry) throws Exception {
String feed =
- Util.setFeedPathValue(bundles[0].getInputFeedFromBundle(), latePath);
+ Util.setFeedPathValue(bundles[0].getInputFeedFromBundle(), lateInputPath);
feed = Util.insertLateFeedValue(feed, getFrequency(retry));
@@ -795,11 +799,11 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
List<String> initialData =
Util.getHadoopDataFromDir(clusterFS, bundles[0].getInputFeedFromBundle(),
- lateDir);
+ lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
String bundleId = OozieUtil.getBundles(clusterOC,
@@ -837,7 +841,7 @@
String insertionFolder =
Util.findFolderBetweenGivenTimeStamps(now, now.plusMinutes(5), initialData);
logger.info("inserting data in folder " + insertionFolder + " at " + DateTime.now());
- HadoopUtil.injectMoreData(clusterFS, lateDir + insertionFolder,
+ HadoopUtil.injectMoreData(clusterFS, lateInputDir + insertionFolder,
OSUtil.OOZIE_EXAMPLE_INPUT_DATA + "lateData");
//now to validate all failed instances to check if they were retried or not.
validateRetry(clusterOC, bundleId,
@@ -854,7 +858,7 @@
public void testRetryInDeleteAfterPartialRetryCase(Retry retry) throws Exception {
String feed =
- Util.setFeedPathValue(bundles[0].getInputFeedFromBundle(), latePath);
+ Util.setFeedPathValue(bundles[0].getInputFeedFromBundle(), lateInputPath);
feed = Util.insertLateFeedValue(feed, new Frequency("minutes(1)"));
bundles[0].getDataSets().remove(bundles[0].getInputFeedFromBundle());
bundles[0].getDataSets().add(feed);
@@ -873,8 +877,8 @@
AssertUtil.assertFailed(response);
} else {
AssertUtil.assertSucceeded(response);
- HadoopUtil.deleteDirIfExists(lateDir, clusterFS);
- HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateDir);
+ HadoopUtil.deleteDirIfExists(lateInputDir, clusterFS);
+ HadoopUtil.lateDataReplenish(clusterFS, 20, 0, lateInputDir);
AssertUtil.assertSucceeded(
prism.getProcessHelper().schedule(bundles[0].getProcessData()));
//now wait till the process is over
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java
index 638768d..cf91d83 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java
@@ -58,6 +58,7 @@
OozieClient clusterOC = serverOC.get(0);
String testDir = baseHDFSDir + "/NoOutputProcessTest";
String inputPath = testDir + "/input" + MINUTE_DATE_PATTERN;
+ String outputPath = testDir + "/output/" + MINUTE_DATE_PATTERN;
String aggregateWorkflowDir = testDir + "/aggregator";
private static final Logger logger = Logger.getLogger(NoOutputProcessTest.class);
@@ -65,7 +66,7 @@
public void createTestData() throws Exception {
logger.info("in @BeforeClass");
HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle b = BundleUtil.readELBundle();
+ Bundle b = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
b.generateUniqueBundle();
b = new Bundle(b, cluster);
String startDate = "2010-01-03T00:00Z";
@@ -80,11 +81,12 @@
@BeforeMethod(alwaysRun = true)
public void testName(Method method) throws Exception {
logger.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0].generateUniqueBundle();
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
bundles[0].setInputFeedDataPath(inputPath);
+ bundles[0].setOutputFeedLocationData(outputPath);
bundles[0].setProcessValidity("2010-01-03T02:30Z", "2010-01-03T02:45Z");
bundles[0].setProcessPeriodicity(5, TimeUnit.minutes);
ProcessMerlin process = new ProcessMerlin(bundles[0].getProcessData());
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java
index cf5ba0f..9a9c686 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java
@@ -75,9 +75,9 @@
LOGGER.info("test name: " + method.getName());
//get 3 unique bundles
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0].generateUniqueBundle();
- bundles[1] = BundleUtil.readELBundle();
+ bundles[1] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[1].generateUniqueBundle();
//generate bundles according to config files
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java
index 7b938ec..68b308e 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java
@@ -66,7 +66,7 @@
public void createTestData() throws Exception {
LOGGER.info("in @BeforeClass");
HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle b = BundleUtil.readELBundle();
+ Bundle b = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
b.generateUniqueBundle();
b = new Bundle(b, cluster);
String startDate = "2010-01-01T23:20Z";
@@ -80,7 +80,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java
index 4ae6d72..900a376 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java
@@ -69,7 +69,8 @@
public void createTestData() throws Exception {
LOGGER.info("in @BeforeClass");
HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle b = BundleUtil.readELBundle();
+ Bundle b = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
+
b = new Bundle(b, cluster);
String startDate = "2010-01-02T00:40Z";
String endDate = "2010-01-02T01:20Z";
@@ -82,7 +83,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setInputFeedDataPath(feedInputPath);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java
index 2e774c2..cfe55eb 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java
@@ -65,7 +65,7 @@
public void createTestData() throws Exception {
LOGGER.info("in @BeforeClass");
HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle b = BundleUtil.readELBundle();
+ Bundle b = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
b = new Bundle(b, cluster);
String startDate = "2010-01-01T23:20Z";
String endDate = "2010-01-02T01:40Z";
@@ -78,7 +78,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
LOGGER.info("setup " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setInputFeedDataPath(feedInputPath);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java
index 7a7e735..8934eb7 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java
@@ -66,7 +66,7 @@
public void createTestData() throws Exception {
LOGGER.info("in @BeforeClass");
HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundle.generateUniqueBundle();
bundle = new Bundle(bundle, cluster);
String startDate = "2010-01-02T00:40Z";
@@ -80,7 +80,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setInputFeedDataPath(feedInputPath);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java
index 12a3907..df959b5 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java
@@ -77,7 +77,8 @@
public void createTestData() throws Exception {
LOGGER.info("in @BeforeClass");
HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle bundle = BundleUtil.readELBundle();
+
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundle.generateUniqueBundle();
bundle = new Bundle(bundle, cluster);
String startDate = "2010-01-01T23:40Z";
@@ -95,7 +96,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setInputFeedDataPath(feedInputPath);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java
index ebe6236..9ba6ceb 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java
@@ -71,7 +71,7 @@
LOGGER.info("in @BeforeClass");
HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundle = new Bundle(bundle, cluster);
String startDate = "2010-01-01T23:40Z";
String endDate = "2010-01-02T01:40Z";
@@ -85,7 +85,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setInputFeedDataPath(feedInputPath);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
index c2d8c9b..743990e 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java
@@ -63,7 +63,7 @@
//common lib for both test cases
HadoopUtil.uploadDir(clusterFS, testLibDir, OSUtil.RESOURCES_OOZIE + "lib");
- Bundle b = BundleUtil.readELBundle();
+ Bundle b = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
b.generateUniqueBundle();
b = new Bundle(b, cluster);
String startDate = "2010-01-01T22:00Z";
@@ -78,7 +78,7 @@
@BeforeMethod(alwaysRun = true)
public void testName(Method method) throws Exception {
logger.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setInputFeedDataPath(baseHDFSDir + MINUTE_DATE_PATTERN);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatFeedOperationsTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatFeedOperationsTest.java
index 8b5c01c..5598744 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatFeedOperationsTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatFeedOperationsTest.java
@@ -93,7 +93,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readHCatBundle();
+ Bundle bundle = BundleUtil.readHCatBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundle, cluster.getPrefix());
bundles[0].generateUniqueBundle();
bundles[0].setClusterInterface(Interfacetype.REGISTRY, cluster.getClusterHelper().getHCatEndpoint());
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
index 980b96f..b3a46a2 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java
@@ -105,7 +105,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp() throws Exception {
clusterHC = cluster.getClusterHelper().getHCatClient();
- bundles[0] = BundleUtil.readHCatBundle();
+ bundles[0] = BundleUtil.readHCatBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(hiveScriptFile, EngineType.HIVE);
@@ -524,7 +524,8 @@
bundles[0].setInputFeedValidity(startDate, endDate);
//
- String nonHCatFeed = BundleUtil.readELBundle().getOutputFeedFromBundle();
+ String nonHCatFeed = BundleUtil.readELBundle(
+ baseAppHDFSDir, this.getClass().getSimpleName()).getOutputFeedFromBundle();
final String outputFeedName = bundles[0].getOutputFeedNameFromBundle();
nonHCatFeed = Util.setFeedName(nonHCatFeed, outputFeedName);
final List<String> clusterNames = bundles[0].getClusterNames();
@@ -577,7 +578,8 @@
.location(outputHDFSDir)
.build());
- String nonHCatFeed = BundleUtil.readELBundle().getInputFeedFromBundle();
+ String nonHCatFeed = BundleUtil.readELBundle(
+ baseAppHDFSDir, this.getClass().getSimpleName()).getInputFeedFromBundle();
final String inputFeedName = bundles[0].getInputFeedNameFromBundle();
nonHCatFeed = Util.setFeedName(nonHCatFeed, inputFeedName);
final List<String> clusterNames = bundles[0].getClusterNames();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
index 50ccd8a..ce94883 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java
@@ -96,7 +96,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp() throws Exception {
- Bundle bundle = BundleUtil.readHCatBundle();
+ Bundle bundle = BundleUtil.readHCatBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundle, cluster.getPrefix());
bundles[0].generateUniqueBundle();
bundles[0].setClusterInterface(Interfacetype.REGISTRY,
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
index aae3125..68924cc 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java
@@ -78,7 +78,8 @@
public void setUp() throws Exception {
HadoopUtil.recreateDir(clusterFS, baseTestHDFSDir);
cli = cluster.getClusterHelper().getHCatClient();
- bundle = new Bundle(BundleUtil.readHCat2Bundle(), cluster);
+ bundle = BundleUtil.readHCat2Bundle(baseAppHDFSDir, this.getClass().getSimpleName());
+ bundle = new Bundle(bundle, cluster);
bundle.generateUniqueBundle();
bundle.submitClusters(prism);
}
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/LineageApiProcessInstanceTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/LineageApiProcessInstanceTest.java
index 5868882..432e682 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/LineageApiProcessInstanceTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/LineageApiProcessInstanceTest.java
@@ -80,7 +80,8 @@
HadoopUtil.deleteDirIfExists(baseTestHDFSDir, clusterFS);
HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- bundles[0] = new Bundle(BundleUtil.readELBundle(), cluster);
+ bundles[0] = new Bundle(
+ BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName()), cluster);
bundles[0].generateUniqueBundle();
bundles[0].setInputFeedDataPath(feedInputPath);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/LineageApiTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/LineageApiTest.java
index 143789f..ed0282e 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/LineageApiTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/LineageApiTest.java
@@ -83,7 +83,7 @@
@BeforeMethod(alwaysRun = true, firstTimeOnly = true)
public void setUp() throws Exception {
CleanupUtil.cleanAllEntities(prism);
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundle.generateUniqueBundle();
bundles[0] = new Bundle(bundle, cluster);
final List<String> clusterStrings = bundles[0].getClusters();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java
index 55d69a3..d3bfc7d 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java
@@ -60,7 +60,7 @@
public void createTestData() throws Exception {
LOGGER.info("in @BeforeClass");
HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle b = BundleUtil.readELBundle();
+ Bundle b = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
b = new Bundle(b, cluster);
b.setInputFeedDataPath(feedInputPath);
}
@@ -68,7 +68,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
LOGGER.info("setup " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setInputFeedDataPath(feedInputPath);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedDelayParallelTimeoutTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedDelayParallelTimeoutTest.java
index 6298981..7e3bcbb 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedDelayParallelTimeoutTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedDelayParallelTimeoutTest.java
@@ -57,7 +57,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundle, cluster1);
bundles[1] = new Bundle(bundle, cluster2);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedRetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedRetentionTest.java
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedRetentionTest.java
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/NewPrismProcessUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/NewPrismProcessUpdateTest.java
index db29b19..1ceab75 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/NewPrismProcessUpdateTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/NewPrismProcessUpdateTest.java
@@ -88,7 +88,7 @@
@BeforeMethod(alwaysRun = true)
public void testSetup(Method method) throws Exception {
LOGGER.info("test name: " + method.getName());
- Bundle b = BundleUtil.readUpdateBundle();
+ Bundle b = BundleUtil.readUpdateBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(b, cluster1);
bundles[0].generateUniqueBundle();
bundles[1] = new Bundle(b, cluster2);
@@ -1451,7 +1451,7 @@
public void
updateProcessWorkflowXml() throws URISyntaxException, JAXBException,
IOException, OozieClientException, AuthenticationException {
- Bundle b = BundleUtil.readELBundle();
+ Bundle b = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
HadoopFileEditor hadoopFileEditor = null;
try {
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/OptionalInputTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/OptionalInputTest.java
index 0c547ab..e1bc38e 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/OptionalInputTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/OptionalInputTest.java
@@ -63,7 +63,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
logger.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismClusterDeleteTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismClusterDeleteTest.java
index c601345..c77efb1 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismClusterDeleteTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismClusterDeleteTest.java
@@ -57,7 +57,8 @@
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
restartRequired = false;
- Bundle bundle = BundleUtil.readLateDataBundle();
+ Bundle bundle = BundleUtil.readLateDataBundle(baseAppHDFSDir,
+ this.getClass().getSimpleName());
bundles[0] = new Bundle(bundle, cluster1);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismConcurrentRequestTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismConcurrentRequestTest.java
index c767649..d4ba566 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismConcurrentRequestTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismConcurrentRequestTest.java
@@ -61,7 +61,7 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws Exception {
logger.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0].generateUniqueBundle();
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedDeleteTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedDeleteTest.java
index f67e2ce..6efbdb0 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedDeleteTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedDeleteTest.java
@@ -69,7 +69,7 @@
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
restartRequired = false;
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundle, cluster1);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedLateReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedLateReplicationTest.java
index 9e0fbed..c16d245 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedLateReplicationTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedLateReplicationTest.java
@@ -67,7 +67,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
for (int i = 0; i < 3; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java
index 7a5a91e..0678595 100755
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java
@@ -173,7 +173,8 @@
@BeforeMethod(alwaysRun = true)
public void testName(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readLocalDCBundle();
+ Bundle bundle =
+ BundleUtil.readLocalDCBundle(baseAppHDFSDir, this.getClass().getSimpleName());
for (int i = 0; i < 3; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java
index 56e0af9..93d82e5 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java
@@ -73,7 +73,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
for (int i = 0; i < 3; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedResumeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedResumeTest.java
index 79ceacc..4902e3f 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedResumeTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedResumeTest.java
@@ -58,7 +58,8 @@
@BeforeMethod(alwaysRun = true)
public void testName(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readLateDataBundle();
+ Bundle bundle = BundleUtil.readLateDataBundle(baseAppHDFSDir,
+ this.getClass().getSimpleName());
for (int i = 0; i < 2; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedScheduleTest.java
index 9a32932..322d66c 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedScheduleTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedScheduleTest.java
@@ -53,7 +53,8 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws IOException {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readLateDataBundle();
+ Bundle bundle = BundleUtil.readLateDataBundle(baseAppHDFSDir,
+ this.getClass().getSimpleName());
for (int i = 0; i < 2; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java
index e97ac37..574131d 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java
@@ -66,7 +66,7 @@
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
restartRequired = false;
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
for (int i = 0; i < 2; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSuspendTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSuspendTest.java
index 1b4ad35..97f41b0 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSuspendTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSuspendTest.java
@@ -56,7 +56,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
for (int i = 0; i < 2; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java
index 9c71375..eae91c6 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java
@@ -76,7 +76,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readELBundle();
+ Bundle bundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
for (int i = 0; i < 2; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessDeleteTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessDeleteTest.java
index 1bc61b3..549a14b 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessDeleteTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessDeleteTest.java
@@ -60,7 +60,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- bundle = BundleUtil.readLateDataBundle();
+ bundle = BundleUtil.readLateDataBundle(baseAppHDFSDir, this.getClass().getSimpleName());
for (int i = 0; i < 2; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessResumeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessResumeTest.java
index c2b9681..3ed7400 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessResumeTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessResumeTest.java
@@ -57,7 +57,8 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readLateDataBundle();
+ Bundle bundle = BundleUtil.readLateDataBundle(baseAppHDFSDir,
+ this.getClass().getSimpleName());
for (int i = 0; i < 2; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java
index e0c25d0..de4af32 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java
@@ -67,7 +67,8 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readLateDataBundle();
+ Bundle bundle = BundleUtil.readLateDataBundle(baseAppHDFSDir,
+ this.getClass().getSimpleName());
for (int i = 0; i < 2; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSnSTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSnSTest.java
index 4768e68..41a8ce5 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSnSTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSnSTest.java
@@ -60,7 +60,8 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readLateDataBundle();
+ Bundle bundle = BundleUtil.readLateDataBundle(baseAppHDFSDir,
+ this.getClass().getSimpleName());
for (int i = 0; i < 2; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSuspendTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSuspendTest.java
index 23e0bd3..45075bc 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSuspendTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSuspendTest.java
@@ -58,7 +58,8 @@
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
restartRequired = false;
- Bundle bundle = BundleUtil.readLateDataBundle();
+ Bundle bundle = BundleUtil.readLateDataBundle(baseAppHDFSDir,
+ this.getClass().getSimpleName());
for (int i = 0; i < 2; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
index b11d586..8c96d7d 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismSubmitTest.java
@@ -60,7 +60,7 @@
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
restartRequired = false;
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster1);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/ProcessPartitionExpVariableTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/ProcessPartitionExpVariableTest.java
index 1a3f82f..b614716 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/ProcessPartitionExpVariableTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/ProcessPartitionExpVariableTest.java
@@ -66,7 +66,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java
index dac4cf6..1bfa5af 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java
@@ -57,7 +57,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleProcessInFinalStatesTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleProcessInFinalStatesTest.java
index d168ed4..928acad 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleProcessInFinalStatesTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleProcessInFinalStatesTest.java
@@ -63,7 +63,7 @@
logger.info("in @BeforeClass");
uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- Bundle b = BundleUtil.readELBundle();
+ Bundle b = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
b.generateUniqueBundle();
b = new Bundle(b, cluster);
b.setProcessWorkflow(aggregateWorkflowDir);
@@ -84,7 +84,7 @@
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
logger.info("test name: " + method.getName());
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setInputFeedDataPath(inputPath);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
index d113554..cd75df8 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
@@ -73,7 +73,7 @@
@BeforeMethod(alwaysRun = true)
public void testName(Method method) throws Exception {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readRetentionBundle();
+ Bundle bundle = BundleUtil.readRetentionBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundle, cluster);
bundles[0].setInputFeedDataPath(testHDFSDir);
bundles[0].generateUniqueBundle();
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/UpdateAtSpecificTimeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/UpdateAtSpecificTimeTest.java
index 378cbbd..71af1e2 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/UpdateAtSpecificTimeTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/UpdateAtSpecificTimeTest.java
@@ -79,7 +79,8 @@
@BeforeMethod(alwaysRun = true)
public void setup(Method method) throws IOException {
logger.info("test name: " + method.getName());
- Bundle bundle = BundleUtil.readLocalDCBundle();
+ Bundle bundle =
+ BundleUtil.readLocalDCBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundle, cluster1);
bundles[1] = new Bundle(bundle, cluster2);
bundles[2] = new Bundle(bundle, cluster3);
@@ -88,7 +89,7 @@
bundles[1].generateUniqueBundle();
bundles[2].generateUniqueBundle();
- processBundle = BundleUtil.readELBundle();
+ processBundle = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
processBundle = new Bundle(processBundle, cluster1);
processBundle.generateUniqueBundle();
processBundle.setProcessWorkflow(aggregateWorkflowDir);
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/LineageGraphTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/LineageGraphTest.java
index d7ac724..b014dad 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/LineageGraphTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/LineageGraphTest.java
@@ -62,7 +62,8 @@
private String baseTestDir = baseHDFSDir + "/LineageGraphTest";
private String aggregateWorkflowDir = baseTestDir + "/aggregator";
private static final Logger logger = Logger.getLogger(LineageGraphTest.class);
- String feedInputPath = baseTestDir + "/input" + MINUTE_DATE_PATTERN;
+ String feedInputPath = baseTestDir + "/input" + MINUTE_DATE_PATTERN;;
+ String feedOutputPath = baseTestDir + "/output" + MINUTE_DATE_PATTERN;;
private FileSystem clusterFS = serverFS.get(0);
private OozieClient clusterOC = serverOC.get(0);
private String processName = null;
@@ -81,7 +82,7 @@
throws IOException, JAXBException, URISyntaxException, AuthenticationException,
OozieClientException {
uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
@@ -95,6 +96,7 @@
bundles[0].setProcessConcurrency(5);
bundles[0].setInputFeedPeriodicity(1, Frequency.TimeUnit.minutes);
bundles[0].setInputFeedDataPath(feedInputPath);
+ bundles[0].setOutputFeedLocationData(feedInputPath);
bundles[0].setProcessInput("now(0,0)", String.format("now(0,%d)", inputEnd - 1));
/**provide necessary data for first 3 instances to run*/
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/ProcessUITest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/ProcessUITest.java
index 76173b7..6f673c9 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/ProcessUITest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ui/ProcessUITest.java
@@ -82,7 +82,7 @@
CleanupUtil.cleanAllEntities(prism);
uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
openBrowser();
- bundles[0] = BundleUtil.readELBundle();
+ bundles[0] = BundleUtil.readELBundle(baseAppHDFSDir, this.getClass().getSimpleName());
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle();
bundles[0].setProcessWorkflow(aggregateWorkflowDir);
@@ -96,6 +96,7 @@
bundles[0].setProcessConcurrency(5);
bundles[0].setInputFeedPeriodicity(1, Frequency.TimeUnit.minutes);
bundles[0].setInputFeedDataPath(feedInputPath + MINUTE_DATE_PATTERN);
+ bundles[0].setOutputFeedLocationData(feedOutputPath + MINUTE_DATE_PATTERN);
Process process = bundles[0].getProcessObject();
Inputs inputs = new Inputs();
Input input = new Input();
diff --git a/falcon-regression/merlin/src/test/resources/hcat/data/_SUCCESS b/falcon-regression/merlin/src/test/resources/hcat/data/_SUCCESS
deleted file mode 100644
index 004c62e..0000000
--- a/falcon-regression/merlin/src/test/resources/hcat/data/_SUCCESS
+++ /dev/null
@@ -1,13 +0,0 @@
-Copyright [yyyy] [name of copyright owner]
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
