OOZIE-3179 Adding a configurable config-default.xml location to a workflow (jphelps via asalamon74)
diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java
index a0f586d..1c6966f 100644
--- a/client/src/main/java/org/apache/oozie/client/OozieClient.java
+++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java
@@ -174,6 +174,8 @@
public static final String LIBPATH = "oozie.libpath";
+ public static final String CONFIG_PATH = "oozie.default.configuration.path";
+
public static final String USE_SYSTEM_LIBPATH = "oozie.use.system.libpath";
public static final String OOZIE_SUSPEND_ON_NODES = "oozie.suspend.on.nodes";
diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java
index 1352db9..c63a24e 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java
@@ -135,30 +135,54 @@
Configuration fsConf = has.createConfiguration(uri.getAuthority());
FileSystem fs = has.createFileSystem(user, uri, fsConf);
- Path configDefault = null;
- Configuration defaultConf = null;
+
+
+ // Generate a list of all default configuration files to be processed
+ // A default-conf.xml in the workflow directory should have higher precedence than service defaults
+ // Adding oozie.default.configuration.path files first will allow default-conf.xml to override duplicates
+ ArrayList<Path> configDefault = new ArrayList<>();
+ Configuration defaultConf = new XConfiguration();
+
+ String[] defaultConfFiles = conf.getStrings(OozieClient.CONFIG_PATH);
+ if (defaultConfFiles != null && defaultConfFiles.length > 0) {
+ for ( String defaultConfFile : defaultConfFiles ) {
+ if (defaultConfFile.trim().length() > 0) {
+ configDefault.add(new Path(defaultConfFile.trim()));
+ }
+
+ }
+ }
+
// app path could be a directory
Path path = new Path(uri.getPath());
if (!fs.isFile(path)) {
- configDefault = new Path(path, CONFIG_DEFAULT);
+ configDefault.add(new Path(path, CONFIG_DEFAULT));
} else {
- configDefault = new Path(path.getParent(), CONFIG_DEFAULT);
+ configDefault.add(new Path(path.getParent(), CONFIG_DEFAULT));
}
- if (fs.exists(configDefault)) {
- try {
- defaultConf = new XConfiguration(fs.open(configDefault));
- PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_USER_PROPERTIES);
- PropertiesUtils.checkDefaultDisallowedProperties(defaultConf);
- XConfiguration.injectDefaults(defaultConf, conf);
+ //
+ for (Path configDefaultFile: configDefault) {
+
+ LOG.debug("Loading Configuration file {0}", configDefaultFile.getName());
+ Configuration defaultConfigs = null;
+ if (fs.exists(configDefaultFile)) {
+ try {
+ defaultConfigs = new XConfiguration(fs.open(configDefaultFile));
+ PropertiesUtils.checkDisallowedProperties(defaultConfigs, DISALLOWED_USER_PROPERTIES);
+ PropertiesUtils.checkDefaultDisallowedProperties(defaultConfigs);
+ XConfiguration.injectDefaults(defaultConfigs, conf);
+
+ }
+ catch (IOException ex) {
+ throw new IOException("Failed Loading default configuration file: " +
+ configDefaultFile.getName() + ex.getMessage(), ex);
+ }
}
- catch (IOException ex) {
- throw new IOException("default configuration file, " + ex.getMessage(), ex);
+ if (defaultConfigs != null) {
+ XConfiguration.copy(resolveDefaultConfVariables(defaultConfigs),defaultConf);
}
}
- if (defaultConf != null) {
- defaultConf = resolveDefaultConfVariables(defaultConf);
- }
WorkflowApp app = wps.parseDef(conf, defaultConf);
XConfiguration protoActionConf = wps.createProtoActionConf(conf, true);
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
index 2cd263f..c27ce23 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java
@@ -421,6 +421,113 @@
assertEquals(getNameNodeUri()+"/default-output-dir", actionConf.get("mixed"));
}
+ public void testWFConfigPathVarResolve() throws Exception {
+ final OozieClient wfClient = LocalOozie.getClient();
+
+ OutputStream os1 = new FileOutputStream(getTestCaseDir() + "/config-default.xml");
+ XConfiguration defaultConf = new XConfiguration();
+ defaultConf.set("outputDir", "default-output-dir");
+ defaultConf.set("should_resolve", "${should.resolve}");
+ defaultConf.set("foo.bar", "default-foo-bar");
+ defaultConf.set("foobarRef", "${foo.bar}");
+ defaultConf.writeXml(os1);
+ os1.close();
+
+ OutputStream os2 = new FileOutputStream(getTestCaseDir() + "/extra-config-file1.xml");
+ XConfiguration confFile1 = new XConfiguration();
+ confFile1.set("outputDir", "default-output-dir_unused1");
+ confFile1.set("should_resolve_file1", "${should.resolve.file1}");
+ confFile1.set("key", "default_value_unused");
+ confFile1.set("foobarRef1", "${foo.bar}");
+ confFile1.writeXml(os2);
+ os2.close();
+
+
+ OutputStream os3 = new FileOutputStream(getTestCaseDir() + "/extra-config-file2.xml");
+ XConfiguration confFile2 = new XConfiguration();
+ confFile2.set("outputDir", "default-output-dir_unused2");
+ confFile2.set("should_resolve_file2", "${should.resolve.file2}");
+ confFile2.set("key", "default_value");
+ confFile2.set("foobarRef2", "${foobarRef1}");
+ confFile2.writeXml(os3);
+ os3.close();
+
+
+ String workflowUri = getTestCaseFileUri("workflow.xml");
+ String actionXml = "<map-reduce>"
+ + "<job-tracker>${jobTracker}</job-tracker>"
+ + "<name-node>${nameNode}</name-node>"
+ + " <prepare>"
+ + " <delete path=\"${nameNode}/user/${wf:user()}/mr/${outputDir}\"/>"
+ + " </prepare>"
+ + " <configuration>"
+ + " <property><name>bb</name><value>BB</value></property>"
+ + " <property><name>cc</name><value>from_action</value></property>"
+ + " </configuration>"
+ + " </map-reduce>";
+ String wfXml = "<workflow-app xmlns=\"uri:oozie:workflow:0.5\" name=\"map-reduce-wf\">"
+ + " <start to=\"mr-node\"/>"
+ + " <action name=\"mr-node\">"
+ + actionXml
+ + " <ok to=\"end\"/>"
+ + " <error to=\"fail\"/>"
+ + "</action>"
+ + "<kill name=\"fail\">"
+ + " <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>"
+ + "</kill>"
+ + "<end name=\"end\"/>"
+ + "</workflow-app>";
+
+ writeToFile(wfXml, workflowUri);
+ Configuration conf = new XConfiguration();
+ conf.set("nameNode", getNameNodeUri());
+ conf.set("jobTracker", getJobTrackerUri());
+ conf.set("foobarRef", "foobarRef");
+ conf.set(OozieClient.APP_PATH, workflowUri);
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ conf.set("should.resolve", "resolved");
+ conf.set("should.resolve.file1", "resolved");
+ conf.set("should.resolve.file2", "resolved");
+ conf.set("oozie.default.configuration.path", getTestCaseDir() + "/extra-config-file1.xml,"
+ + getTestCaseDir() + "/extra-config-file2.xml");
+ SubmitXCommand sc = new SubmitXCommand(conf);
+ final String jobId = sc.call();
+ new StartXCommand(jobId).call();
+ waitFor(15 * 1000, new Predicate() {
+ public boolean evaluate() throws Exception {
+ return wfClient.getJobInfo(jobId).getStatus() == WorkflowJob.Status.KILLED;
+ }
+ });
+ String actionId = jobId + "@mr-node";
+ WorkflowActionBean action = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQueryExecutor.WorkflowActionQuery
+ .GET_ACTION, actionId);
+ Element eAction = XmlUtils.parseXml(action.getConf());
+ Element eConf = eAction.getChild("configuration", eAction.getNamespace());
+ Configuration actionConf = new XConfiguration(new StringReader(XmlUtils.prettyPrint(eConf).toString()));
+ assertEquals("Variable should_resolve!='resolved' from config-default.xml",
+ "resolved", actionConf.get("should_resolve"));
+ assertEquals("Variable 'should_resolve_file1' from extra-config-file1 did not resolve correctly",
+ "resolved", actionConf.get("should_resolve_file1"));
+ assertEquals("Variable 'should_resolve_file2' from extra-config-file2 did not resolve correctly",
+ "resolved", actionConf.get("should_resolve_file2"));
+ assertEquals("Variable 'outputDir' from config-default did not resolve correctly",
+ "default-output-dir", actionConf.get("outputDir"));
+ assertEquals("Variable 'bb' from workflow did not resolve correctly",
+ "BB", actionConf.get("bb"));
+ assertEquals("Variable 'cc' from workflow did not resolve correctly",
+ "from_action", actionConf.get("cc"));
+ assertEquals("Variable 'foo.bar' from config-default did not resolve correctly",
+ "default-foo-bar", actionConf.get("foo.bar"));
+ assertEquals("Variable 'foobarRef' from config-default did not resolve correctly",
+ "default-foo-bar", actionConf.get("foobarRef"));
+ assertEquals("Variable 'foobarRef1' from extra-config-file1 did not resolve correctly",
+ "default-foo-bar", actionConf.get("foobarRef1"));
+ assertEquals("Variable 'foobarRef2' from extra-config-file2 did not resolve correctly",
+ "default-foo-bar", actionConf.get("foobarRef2"));
+ assertEquals("Variable 'key' from extra-config-file2 did not resolve correctly",
+ "default_value", actionConf.get("key"));
+ }
+
private void writeToFile(String appXml, String appPath) throws IOException {
File wf = new File(URI.create(appPath));
diff --git a/release-log.txt b/release-log.txt
index c5cbc5f..d777a38 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.2.0 release (trunk - unreleased)
+OOZIE-3179 Adding a configurable config-default.xml location to a workflow (jphelps via asalamon74)
OOZIE-3405 SSH action shows empty error Message and Error code (matijhs via asalamon74)
OOZIE-3542 Handle better old Hdfs implementations in ECPolicyDisabler (zsombor dionusos via kmarton)
OOZIE-3540 Use StringBuilder instead of StringBuffer if concurrent access is not required (zsombor via asalamon74)
