[OODT-984] Logging improvements for PGE and fixed NPE in workflowInstanceId in PGETaskInstance
diff --git a/pge/src/main/java/org/apache/oodt/cas/pge/PGETask.java b/pge/src/main/java/org/apache/oodt/cas/pge/PGETask.java
index 906b0e5..d55c532 100644
--- a/pge/src/main/java/org/apache/oodt/cas/pge/PGETask.java
+++ b/pge/src/main/java/org/apache/oodt/cas/pge/PGETask.java
@@ -20,10 +20,11 @@
import org.apache.oodt.cas.metadata.SerializableMetadata;
import org.apache.oodt.cas.workflow.structs.WorkflowTaskConfiguration;
import org.apache.oodt.cas.workflow.structs.exceptions.WorkflowTaskInstanceException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.util.logging.Logger;
import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.createPGETaskInstance;
@@ -37,7 +38,7 @@
*/
public class PGETask {
- private static final Logger LOGGER = Logger.getLogger(PGETask.class.getName());
+ private static final Logger logger = LoggerFactory.getLogger(PGETask.class);
private Metadata metadata;
@@ -51,8 +52,7 @@
public void run(String pgeTaskInstanceClasspath)
throws
WorkflowTaskInstanceException {
- PGETaskInstance pgeTaskInst = createPGETaskInstance(
- pgeTaskInstanceClasspath, LOGGER);
+ PGETaskInstance pgeTaskInst = createPGETaskInstance(pgeTaskInstanceClasspath, logger);
pgeTaskInst.run(this.metadata, this.wftConfig);
}
diff --git a/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java b/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
index 2e266e7..2609df1 100644
--- a/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
+++ b/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
@@ -57,6 +57,8 @@
import org.apache.oodt.cas.workflow.util.ScriptFile;
import org.apache.oodt.commons.exceptions.CommonsException;
import org.apache.oodt.commons.exec.ExecUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.context.support.FileSystemXmlApplicationContext;
import java.io.File;
@@ -71,8 +73,6 @@
import java.util.LinkedList;
import java.util.List;
import java.util.logging.FileHandler;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
import java.util.regex.Pattern;
@@ -96,7 +96,14 @@
*/
public class PGETaskInstance implements WorkflowTaskInstance {
- protected Logger logger = Logger.getLogger(PGETaskInstance.class.getName());
+ protected Logger logger = LoggerFactory.getLogger(PGETaskInstance.class);
+
+ /**
+ * This JUL logger is kept for now to avoid large scale changes in logging dependencies.
+ * Should be removed in future with an alternative to address the logging dependencies.
+ */
+ protected java.util.logging.Logger julLogger = java.util.logging.Logger.getLogger(PGETaskInstance.class.getName());
+
private WorkflowManagerClient wmClient;
private String workflowInstId;
protected PgeMetadata pgeMetadata;
@@ -106,12 +113,18 @@
@Override
public void run(Metadata metadata, WorkflowTaskConfiguration config) throws WorkflowTaskInstanceException {
+ logger.debug("Starting PGE Task instance...");
try {
// Initialize CAS-PGE.
pgeMetadata = createPgeMetadata(metadata, config);
pgeConfig = createPgeConfig();
runPropertyAdders();
- logger = createLogger(); // use workflow ID specific logger from now on
+ workflowInstId = getWorkflowInstanceId();
+
+ // use workflow ID specific logger from now on
+ logger = LoggerFactory.getLogger(PGETaskInstance.class.getName() + "." + workflowInstId);
+ logger.debug("Workflow instance ID is [{}]", workflowInstId);
+ julLogger = createLogger();
// Write out PgeMetadata.
dumpMetadataIfRequested();
@@ -135,30 +148,27 @@
// Commit dynamic metadata.
updateDynamicMetadata();
} catch (Exception e) {
- logger.log(Level.SEVERE, "PGETask FAILED!!! : " + e.getMessage(), e);
- throw new WorkflowTaskInstanceException("PGETask FAILED!!! : "
- + e.getMessage(), e);
+ logger.error("PGETask FAILED!!! Error occurred when running", e);
+ throw new WorkflowTaskInstanceException("PGETask FAILED!!! : " + e.getMessage(), e);
}
}
protected void updateStatus(String status) throws Exception {
logger.info("Updating status to workflow as [" + status + "]");
if (!getWorkflowManagerClient().updateWorkflowInstanceStatus(workflowInstId, status)) {
- throw new PGEException(
- "Failed to update workflow status : client returned false");
+ throw new PGEException("Failed to update workflow status : client returned false");
}
}
- protected Logger createLogger() throws IOException, PGEException {
+ protected java.util.logging.Logger createLogger() throws IOException, PGEException {
File logDir = new File(pgeConfig.getExeDir(), "logs");
if (!(logDir.exists() || logDir.mkdirs())) {
throw new PGEException("mkdirs for logs directory return false");
}
- Logger logger = Logger.getLogger(PGETaskInstance.class.getName()
- + "." + workflowInstId);
- FileHandler handler = new FileHandler(
- new File(logDir, createLogFileName()).getAbsolutePath());
+ java.util.logging.Logger logger = java.util.logging.Logger.getLogger(PGETaskInstance.class.getName() + "." + workflowInstId);
+ // TODO Need to find an alternative way to add a dynamic handler to write workflowInstance logs to a separate file
+ FileHandler handler = new FileHandler(new File(logDir, createLogFileName()).getAbsolutePath());
handler.setEncoding("UTF-8");
handler.setFormatter(new SimpleFormatter());
logger.addHandler(handler);
@@ -170,14 +180,14 @@
if (filenamePattern != null) {
return filenamePattern;
} else {
- return pgeMetadata.getMetadata(NAME) + "." + System.currentTimeMillis()
- + ".log";
+ return pgeMetadata.getMetadata(NAME) + "." + System.currentTimeMillis() + ".log";
}
}
- protected PgeMetadata createPgeMetadata(Metadata dynMetadata,
- WorkflowTaskConfiguration config) {
- logger.info("Converting workflow configuration to static metadata...");
+ protected PgeMetadata createPgeMetadata(Metadata dynMetadata, WorkflowTaskConfiguration config) {
+ logger.info("Converting workflow configuration to static metadata");
+ logger.debug("PGE Metadata Config: {}", config.getProperties().entrySet());
+ logger.debug("PGE Metadata Dynamic Metadata: {}", dynMetadata.getMap());
Metadata staticMetadata = new Metadata();
for (Object objKey : config.getProperties().keySet()) {
String key = (String) objKey;
@@ -187,36 +197,30 @@
Splitter.on(",").trimResults()
.omitEmptyStrings()
.split(config.getProperty(key)));
- logger.finest("Adding static metadata: key = [" + key
- + "] value = " + values);
+ logger.debug("Adding static metadata: key = [{}] value = {}", key, values);
staticMetadata.addMetadata(key, values);
} else {
String value = config.getProperty(key);
- logger.finest("Adding static metadata: key = [" + key
- + "] value = [" + value + "]");
+ logger.debug("Adding static metadata: key = [{}] value = {}", key, value);
staticMetadata.addMetadata(key, value);
}
}
+
logger.info("Loading workflow context metadata...");
for (String key : dynMetadata.getAllKeys()) {
- logger.finest(
- "Adding dynamic metadata: key = [" + key + "] value = "
- + dynMetadata.getAllMetadata(key));
+ logger.debug("Adding dynamic metadata: key = [{}] value = {}", key, dynMetadata.getAllMetadata(key));
}
return new PgeMetadata(staticMetadata, dynMetadata);
}
protected PgeConfig createPgeConfig() throws Exception {
logger.info("Create PgeConfig...");
- String pgeConfigBuilderClass = pgeMetadata
- .getMetadata(PGE_CONFIG_BUILDER);
+ String pgeConfigBuilderClass = pgeMetadata.getMetadata(PGE_CONFIG_BUILDER);
if (pgeConfigBuilderClass != null) {
- logger.info("Using PgeConfigBuilder: " + pgeConfigBuilderClass);
- return createPgeConfigBuilder(pgeConfigBuilderClass, logger)
- .build(pgeMetadata);
+ logger.info("Using PgeConfigBuilder: {}", pgeConfigBuilderClass);
+ return createPgeConfigBuilder(pgeConfigBuilderClass, logger).build(pgeMetadata);
} else {
- logger.info("Using default PgeConfigBuilder: "
- + XmlFilePgeConfigBuilder.class.getCanonicalName());
+ logger.info("Using default PgeConfigBuilder: {}", XmlFilePgeConfigBuilder.class.getCanonicalName());
return new XmlFilePgeConfigBuilder().build(pgeMetadata);
}
}
@@ -224,8 +228,7 @@
protected void runPropertyAdders() throws PGEException {
try {
logger.info("Loading/Running property adders...");
- List<String> propertyAdders = pgeMetadata
- .getAllMetadata(PROPERTY_ADDERS);
+ List<String> propertyAdders = pgeMetadata.getAllMetadata(PROPERTY_ADDERS);
if (propertyAdders != null) {
for (String propertyAdder : propertyAdders) {
runPropertyAdder(loadPropertyAdder(propertyAdder));
@@ -234,22 +237,19 @@
logger.info("No property adders specified");
}
} catch (Exception e) {
- throw new PGEException("Failed to instanciate/run Property Adders : "
- + e.getMessage(), e);
+ logger.error("Error occurred when running property adders", e);
+ throw new PGEException("Failed to instantiate/run Property Adders : " + e.getMessage(), e);
}
}
- protected ConfigFilePropertyAdder loadPropertyAdder(
- String propertyAdderClasspath) {
- logger.fine("Loading property adder: " + propertyAdderClasspath);
+ protected ConfigFilePropertyAdder loadPropertyAdder(String propertyAdderClasspath) {
+ logger.debug("Loading property adder: {}", propertyAdderClasspath);
return createConfigFilePropertyAdder(propertyAdderClasspath, logger);
}
protected void runPropertyAdder(ConfigFilePropertyAdder propAdder) {
- logger.info("Running property adder: "
- + propAdder.getClass().getCanonicalName());
- propAdder.addConfigProperties(pgeMetadata,
- pgeConfig.getPropertyAdderCustomArgs());
+ logger.info("Running property adder: {}", propAdder.getClass().getCanonicalName());
+ propAdder.addConfigProperties(pgeMetadata, pgeConfig.getPropertyAdderCustomArgs());
}
protected WorkflowManagerClient getWorkflowManagerClient() throws MalformedURLException {
@@ -265,9 +265,8 @@
protected String getWorkflowInstanceId() {
String instanceId = pgeMetadata.getMetadata(CoreMetKeys.WORKFLOW_INST_ID);
- logger.info("Workflow instanceId is [" + instanceId + "]");
- Validate.notNull(instanceId, "Must specify "
- + CoreMetKeys.WORKFLOW_INST_ID);
+ logger.debug("Workflow instanceId is [{}]", instanceId);
+ Validate.notNull(instanceId, "Must specify " + CoreMetKeys.WORKFLOW_INST_ID);
return instanceId;
}
@@ -290,24 +289,22 @@
}
protected void createExeDir() throws PGEException {
- logger.info("Creating PGE execution working directory: ["
- + pgeConfig.getExeDir() + "]");
+ logger.info("Creating PGE execution working directory: [{}]", pgeConfig.getExeDir());
File executionDir = new File(pgeConfig.getExeDir());
if (!(executionDir.exists() || executionDir.mkdirs())) {
- throw new PGEException("mkdirs returned false for creating ["
- + pgeConfig.getExeDir() + "]");
+ logger.warn("Unable to create execution working directory: {}", pgeConfig.getExeDir());
+ throw new PGEException("mkdirs returned false for creating [" + pgeConfig.getExeDir() + "]");
}
}
protected void createOuputDirsIfRequested() throws PGEException {
for (OutputDir outputDir : pgeConfig.getOuputDirs()) {
if (outputDir.isCreateBeforeExe()) {
- logger.info("Creating PGE file ouput directory: ["
- + outputDir.getPath() + "]");
+ logger.info("Creating PGE file ouput directory: [{}]", outputDir.getPath());
File dir = new File(outputDir.getPath());
if (!(dir.exists() || dir.mkdirs())) {
- throw new PGEException("mkdir returned false for creating ["
- + outputDir.getPath() + "]");
+ logger.warn("Unable to create output dirs: {}", outputDir.getPath());
+ throw new PGEException("mkdir returned false for creating [" + outputDir.getPath() + "]");
}
}
}
@@ -319,8 +316,7 @@
if (pgeConfig.getFileStagingInfo() != null) {
FileStager fileStager = getFileStager();
logger.info("Starting file staging...");
- fileStager.stageFiles(
- pgeConfig.getFileStagingInfo(), pgeMetadata, logger);
+ fileStager.stageFiles(pgeConfig.getFileStagingInfo(), pgeMetadata, logger);
} else {
logger.info("No files to stage.");
}
@@ -329,7 +325,7 @@
protected FileStager getFileStager() {
String fileStagerClass = pgeMetadata.getMetadata(FILE_STAGER);
if (fileStagerClass != null) {
- logger.info("Loading FileStager [" + fileStagerClass + "]");
+ logger.info("Loading FileStager [{}]", fileStagerClass);
return createFileStager(fileStagerClass, logger);
} else {
logger.info("Using default FileStager ["
@@ -350,38 +346,33 @@
protected void createDynamicConfigFile(DynamicConfigFile dynamicConfigFile)
throws PGEException, IOException {
Validate.notNull(dynamicConfigFile, "dynamicConfigFile cannot be null");
- logger.fine("Starting creation of sci pge config file ["
- + dynamicConfigFile.getFilePath() + "]...");
+ logger.debug("Starting creation of sci pge config file: {}", dynamicConfigFile.getFilePath());
// Create parent directory if it doesn't exist.
- File parentDir = new File(dynamicConfigFile.getFilePath())
- .getParentFile();
+ File parentDir = new File(dynamicConfigFile.getFilePath()).getParentFile();
if (!(parentDir.exists() || parentDir.mkdirs())) {
+ logger.warn("Unable to create directory({}) with sci pge config file", dynamicConfigFile.getFilePath());
throw new PGEException("Failed to create directory where sci pge config file ["
+ dynamicConfigFile.getFilePath() + "] was to be written");
}
// Load writer and write file.
- logger.fine("Loading writer class for sci pge config file ["
- + dynamicConfigFile.getFilePath() + "]...");
- SciPgeConfigFileWriter writer = createSciPgeConfigFileWriter(
- dynamicConfigFile.getWriterClass(), logger);
- logger.fine("Loaded writer [" + writer.getClass().getCanonicalName()
- + "] for sci pge config file [" + dynamicConfigFile.getFilePath()
- + "]...");
- logger.info("Writing sci pge config file [" + dynamicConfigFile.getFilePath()
- + "]...");
+ logger.debug("Loading writer class for sci pge config file [{}]", dynamicConfigFile.getFilePath());
+ SciPgeConfigFileWriter writer = createSciPgeConfigFileWriter(dynamicConfigFile.getWriterClass(), logger);
+ logger.debug("Loaded writer [{}] for sci pge config file [{}]",
+ writer.getClass().getCanonicalName(),dynamicConfigFile.getFilePath());
+ logger.info("Writing sci pge config file [{}]", dynamicConfigFile.getFilePath());
File configFile = writer.createConfigFile(dynamicConfigFile.getFilePath(),
- pgeMetadata.asMetadata(), dynamicConfigFile.getArgs());
+ pgeMetadata.asMetadata(), dynamicConfigFile.getArgs());
if (!configFile.exists()) {
- throw new PGEException("Writer failed to create config file ["
- + configFile + "], exists returned false");
+ logger.warn("Failed to create config file '{}'. File doesn't exist", configFile);
+ throw new PGEException("Writer failed to create config file [" + configFile + "], exists returned false");
}
}
protected ScriptFile buildPgeRunScript() {
- logger.fine("Creating PGE run script for shell [" + pgeConfig.getShellType()
- + "] with contents " + pgeConfig.getExeCmds());
+ logger.debug("Creating PGE run script for shell [{}] with contents: {}",
+ pgeConfig.getShellType(), pgeConfig.getExeCmds());
ScriptFile sf = new ScriptFile(pgeConfig.getShellType());
sf.setCommands(pgeConfig.getExeCmds());
return sf;
@@ -389,13 +380,13 @@
protected File getScriptPath() {
File script = new File(pgeConfig.getExeDir(), getPgeScriptName());
- logger.fine("Script file with be written to [" + script + "]");
+ logger.debug("Script file with be written to [{}]", script);
return script;
}
protected String getPgeScriptName() {
String pgeScriptName = "sciPgeExeScript_" + pgeMetadata.getMetadata(NAME);
- logger.fine("Generated script file name [" + pgeScriptName + "]");
+ logger.debug("Generated script file name [{}]", pgeScriptName);
return pgeScriptName;
}
@@ -403,7 +394,7 @@
ScriptFile sf = null;
try {
long startTime = System.currentTimeMillis();
- logger.info("PGE start time [" + new Date(startTime) + "]");
+ logger.info("PGE start time [{}]", new Date(startTime));
// create script to run
sf = buildPgeRunScript();
@@ -411,69 +402,67 @@
// run script and evaluate whether success or failure
updateStatus(RUNNING_PGE.getWorkflowStatusName());
- logger.info("Starting execution of PGE...");
+ logger.debug("Starting execution of PGE: {}", sf.getCommands());
if (!wasPgeSuccessful(ExecUtils.callProgram(
- pgeConfig.getShellType() + " " + getScriptPath(), logger,
+ pgeConfig.getShellType() + " " + getScriptPath(), julLogger,
new File(pgeConfig.getExeDir()).getAbsoluteFile()))) {
+ logger.error("PGE didn't finish successfully: {}", sf);
throw new RuntimeException("Pge didn't finish successfully");
} else {
- logger.info(
- "Successfully completed running: '" + sf.getCommands() + "'");
+ logger.info("Successfully completed running script file: '{}'", sf);
}
long endTime = System.currentTimeMillis();
- logger.info("PGE end time [" + new Date(startTime) + "]");
+ logger.info("PGE end time [{}]", new Date(startTime));
long runTime = endTime - startTime;
- logger.info("PGE runtime in millis [" + runTime + "]");
+ logger.info("PGE runtime in millis [{}]", runTime);
pgeMetadata.replaceMetadata(PGE_RUNTIME, Long.toString(runTime));
-
- } catch (WorkflowException e) {
- throw new PGEException("Exception when executing PGE commands '" + (sf.getCommands()) + "' : " + e.getMessage(), e);
- } catch (IOException e) {
- throw new PGEException("Exception when executing PGE commands '" + (sf.getCommands()) + "' : " + e.getMessage(), e);
+ } catch (WorkflowException | IOException e) {
+ logger.error("Error when executing PGE commands: {}", sf);
+ throw new PGEException("Exception when executing PGE commands '" +
+ (sf.getCommands()) + "' : " + e.getMessage(), e);
}
}
protected boolean wasPgeSuccessful(int returnCode) {
return returnCode == 0;
}
-
+
protected void processOutput() throws IOException {
- for (final OutputDir outputDir : this.pgeConfig.getOuputDirs()) {
+ logger.debug("Processing output");
+ for (final OutputDir outputDir : this.pgeConfig.getOuputDirs()) {
File[] createdFiles = new File(outputDir.getPath()).listFiles();
- if (createdFiles != null) {
- for (File createdFile : createdFiles) {
- Metadata outputMetadata = new Metadata();
- for (RegExprOutputFiles regExprFiles : outputDir
- .getRegExprOutputFiles()) {
- if (Pattern.matches(regExprFiles.getRegExp(), createdFile
- .getName())) {
+ if (createdFiles != null) {
+ for (File createdFile : createdFiles) {
+ Metadata outputMetadata = new Metadata();
+ for (RegExprOutputFiles regExprFiles : outputDir
+ .getRegExprOutputFiles()) {
+ if (Pattern.matches(regExprFiles.getRegExp(), createdFile
+ .getName())) {
try {
- PcsMetFileWriter writer = (PcsMetFileWriter) Class
- .forName(regExprFiles.getConverterClass())
- .newInstance();
- outputMetadata.replaceMetadata(this.getMetadataForFile(
- (regExprFiles.getRenamingConv() != null)
- ? createdFile = this.renameFile(createdFile, regExprFiles.getRenamingConv())
- : createdFile, writer, regExprFiles.getArgs()));
+ PcsMetFileWriter writer = (PcsMetFileWriter) Class
+ .forName(regExprFiles.getConverterClass())
+ .newInstance();
+ outputMetadata.replaceMetadata(this.getMetadataForFile(
+ (regExprFiles.getRenamingConv() != null)
+ ? createdFile = this.renameFile(createdFile, regExprFiles.getRenamingConv())
+ : createdFile, writer, regExprFiles.getArgs()));
} catch (Exception e) {
- logger.severe(
- "Failed to create metadata file for '"
- + createdFile + "' : "
- + e.getMessage());
+ logger.error("Failed to create metadata file for '{}'", createdFile, e);
}
- }
- }
- if (outputMetadata.getAllKeys().size() > 0) {
- this.writeFromMetadata(outputMetadata, createdFile.getAbsolutePath()
- + "." + this.pgeMetadata.getMetadata(MET_FILE_EXT));
- }
+ }
+ }
+
+ if (outputMetadata.getAllKeys().size() > 0) {
+ this.writeFromMetadata(outputMetadata, createdFile.getAbsolutePath()
+ + "." + this.pgeMetadata.getMetadata(MET_FILE_EXT));
+ }
+ }
}
- }
- }
- }
+ }
+ }
protected File renameFile(File file, PathUtilsNamingConvention renamingConv)
throws NamingConventionException {
@@ -496,29 +485,28 @@
.writeMetadataToXmlStream(new FileOutputStream(toMetFilePath));
}
- protected ProductCrawler createProductCrawler()
- throws MalformedURLException, IllegalAccessException, CrawlerActionException, MetExtractionException,
- InstantiationException, FileNotFoundException, ClassNotFoundException {
- /* create a ProductCrawler based on whether or not the output dir specifies a MIME_EXTRACTOR_REPO */
+ protected ProductCrawler createProductCrawler()
+ throws MalformedURLException, IllegalAccessException, CrawlerActionException, MetExtractionException,
+ InstantiationException, FileNotFoundException, ClassNotFoundException {
+ /* create a ProductCrawler based on whether or not the output dir specifies a MIME_EXTRACTOR_REPO */
logger.info("Configuring ProductCrawler...");
ProductCrawler crawler;
- if (pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO) != null &&
- !pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO).equals("")){
- crawler = new AutoDetectProductCrawler();
- ((AutoDetectProductCrawler)crawler).
- setMimeExtractorRepo(pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO));
- }
- else{
- crawler = new StdProductCrawler();
+ if (pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO) != null &&
+ !pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO).equals("")) {
+ crawler = new AutoDetectProductCrawler();
+ ((AutoDetectProductCrawler) crawler).
+ setMimeExtractorRepo(pgeMetadata.getMetadata(MIME_EXTRACTOR_REPO));
+ } else {
+ crawler = new StdProductCrawler();
}
crawler.setClientTransferer(pgeMetadata
- .getMetadata(INGEST_CLIENT_TRANSFER_SERVICE_FACTORY));
+ .getMetadata(INGEST_CLIENT_TRANSFER_SERVICE_FACTORY));
crawler.setFilemgrUrl(pgeMetadata.getMetadata(INGEST_FILE_MANAGER_URL));
String crawlerConfigFile = pgeMetadata.getMetadata(CRAWLER_CONFIG_FILE);
if (!Strings.isNullOrEmpty(crawlerConfigFile)) {
crawler.setApplicationContext(
- new FileSystemXmlApplicationContext(crawlerConfigFile));
+ new FileSystemXmlApplicationContext(crawlerConfigFile));
List<String> actionIds = pgeMetadata.getAllMetadata(ACTION_IDS);
if (actionIds != null) {
crawler.setActionIds(actionIds);
@@ -526,14 +514,12 @@
}
crawler.setRequiredMetadata(pgeMetadata.getAllMetadata(REQUIRED_METADATA));
crawler.setCrawlForDirs(Boolean.parseBoolean(pgeMetadata
- .getMetadata(CRAWLER_CRAWL_FOR_DIRS)));
+ .getMetadata(CRAWLER_CRAWL_FOR_DIRS)));
crawler.setNoRecur(!Boolean.parseBoolean(
- pgeMetadata.getMetadata(CRAWLER_RECUR)));
- logger.fine(
- "Passing Workflow Metadata to CAS-Crawler as global metadata . . .");
+ pgeMetadata.getMetadata(CRAWLER_RECUR)));
+ logger.debug("Passing Workflow Metadata to CAS-Crawler as global metadata . . .");
crawler.setGlobalMetadata(pgeMetadata.asMetadata(PgeMetadata.Type.DYNAMIC));
- logger.fine("Created ProductCrawler ["
- + crawler.getClass().getCanonicalName() + "]");
+ logger.debug("Created ProductCrawler [{}]", crawler.getClass().getCanonicalName());
return crawler;
}
@@ -566,7 +552,7 @@
}
protected void verifyIngests(ProductCrawler crawler) throws PGEException {
- logger.info("Verifying ingests successful...");
+ logger.debug("Verifying ingests successful...");
boolean ingestsSuccess = true;
String exceptionMsg = "";
for (IngestStatus status : crawler.getIngestStatus()) {
@@ -577,12 +563,13 @@
+ status.getResult() + "',msg='" + status.getMessage() + "']";
ingestsSuccess = false;
} else if (!status.getResult().equals(IngestStatus.Result.SUCCESS)) {
- logger.warning("Product was not ingested [file='"
- + status.getProduct().getAbsolutePath() + "',result='"
- + status.getResult() + "',msg='" + status.getMessage() + "']");
+ logger.warn("Product was not ingested [file='{}', result='{}', msg='{}'",
+ status.getProduct().getAbsolutePath(), status.getResult(),status.getMessage());
}
}
+
if (!ingestsSuccess) {
+ logger.error("Ingest wasn't successful: {}", exceptionMsg);
throw new PGEException(exceptionMsg);
} else {
logger.info("Ingests were successful");
@@ -590,6 +577,7 @@
}
protected void updateDynamicMetadata() throws Exception {
+ logger.debug("Updating dynamic metadata ...");
pgeMetadata.commitMarkedDynamicMetadataKeys();
getWorkflowManagerClient()
.updateMetadataForWorkflow(workflowInstId, pgeMetadata.asMetadata(PgeMetadata.Type.DYNAMIC));
@@ -600,6 +588,7 @@
}
public void setWorkflowInstId(String workflowInstId) {
+ logger.debug("Set workflow instance ID: {}", workflowInstId);
this.workflowInstId = workflowInstId;
}
@@ -609,8 +598,10 @@
@Override
public void finalize() throws IOException {
+ logger.debug("Finalizing ...");
if (wmClient != null) {
wmClient.close();
+ logger.debug("Workflow manager client closed");
}
}
}
diff --git a/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileManagerFileStager.java b/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileManagerFileStager.java
index dbf6602..4cbd275 100644
--- a/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileManagerFileStager.java
+++ b/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileManagerFileStager.java
@@ -32,6 +32,7 @@
//Google imports
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
+import org.slf4j.Logger;
//JDK imports
import java.io.File;
@@ -39,8 +40,6 @@
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
-import java.util.logging.Level;
-import java.util.logging.Logger;
/**
* A {@link FileStager} which uses a FileManager {@link DataTransferer}.
@@ -53,8 +52,7 @@
public void stageFile(URI stageFile, File destDir,
PgeMetadata pgeMetadata, Logger logger) throws IOException, DataTransferException, InstantiationException {
DataTransfer dataTransferer = createDataTransfer(pgeMetadata, logger);
- logger.log(Level.INFO, "Using DataTransfer ["
- + dataTransferer.getClass().getCanonicalName() + "]");
+ logger.info("Using DataTransfer [{}]", dataTransferer.getClass().getCanonicalName());
setFileManagerUrl(dataTransferer, pgeMetadata, logger);
dataTransferer.retrieveProduct(createProduct(stageFile), destDir);
}
@@ -67,7 +65,7 @@
.getDataTransferServiceFromFactory(pgeMetadata
.getMetadata(QUERY_CLIENT_TRANSFER_SERVICE_FACTORY));
} else {
- logger.log(Level.INFO, "Using default DataTransferer");
+ logger.info("Using default DataTransferer");
return new RemoteDataTransferFactory().createDataTransfer();
}
}
@@ -79,9 +77,8 @@
if (filemgrUrl != null) {
dataTransferer.setFileManagerUrl(new URL(filemgrUrl));
} else {
- logger.log(Level.WARNING, "Metadata field [" + QUERY_FILE_MANAGER_URL
- + "] was not set, if DataTranferer requires filemgr server,"
- + " your transfers will fail");
+ logger.warn("Metadata field [{}] was not set, if DataTranferer requires filemgr server, your transfers will fail",
+ QUERY_FILE_MANAGER_URL);
}
}
diff --git a/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java b/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java
index ab98823..28bb24c 100644
--- a/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java
+++ b/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java
@@ -30,6 +30,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
+import org.slf4j.Logger;
import java.io.File;
import java.io.IOException;
@@ -38,8 +39,6 @@
import java.net.URISyntaxException;
import java.net.URL;
import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.QUERY_FILE_MANAGER_URL;
@@ -51,36 +50,26 @@
*/
public abstract class FileStager {
- public void stageFiles(FileStagingInfo fileStagingInfo,
- PgeMetadata pgeMetadata, Logger logger)
+ public void stageFiles(FileStagingInfo fileStagingInfo, PgeMetadata pgeMetadata, Logger logger)
throws PGEException, CatalogException, URISyntaxException, IOException, ConnectionException,
InstantiationException, DataTransferException {
- logger.log(Level.INFO, "Creating staging directory ["
- + fileStagingInfo.getStagingDir() + "]");
+ logger.info("Creating staging directory [{}]", fileStagingInfo.getStagingDir());
new File(fileStagingInfo.getStagingDir()).mkdirs();
for (String file : fileStagingInfo.getFilePaths()) {
File fileHandle = new File(file);
if (fileStagingInfo.isForceStaging() || !fileHandle.exists()) {
- logger.log(Level.INFO, "Staging file [" + file
- + "] to directory ["
- + fileStagingInfo.getStagingDir() + "]");
- stageFile(asURI(file), new File(fileStagingInfo.getStagingDir()),
- pgeMetadata, logger);
+ logger.info("Staging file [{}] to directory [{}]", file, fileStagingInfo.getStagingDir());
+ stageFile(asURI(file), new File(fileStagingInfo.getStagingDir()), pgeMetadata, logger);
}
}
if (!fileStagingInfo.getProductIds().isEmpty()) {
FileManagerClient fmClient = createFileManagerClient(pgeMetadata);
for (String productId : fileStagingInfo.getProductIds()) {
- logger.log(Level.INFO, "Staging product [" + productId
- + "] to directory ["
- + fileStagingInfo.getStagingDir() + "]");
+ logger.info("Staging product [{}] to directory [{}]", productId, fileStagingInfo.getStagingDir());
for (URI uri : getProductReferences(productId, fmClient)) {
- logger.log(Level.INFO, "Staging product [" + productId
- + "] reference [" + uri
- + "] to directory ["
- + fileStagingInfo.getStagingDir() + "]");
- stageFile(uri, new File(fileStagingInfo.getStagingDir()),
- pgeMetadata, logger);
+ logger.info("Staging product [{}] reference [{}] to directory [{}]",
+ productId, uri, fileStagingInfo.getStagingDir());
+ stageFile(uri, new File(fileStagingInfo.getStagingDir()), pgeMetadata, logger);
}
}
}
diff --git a/pge/src/main/java/org/apache/oodt/cas/pge/util/GenericPgeObjectFactory.java b/pge/src/main/java/org/apache/oodt/cas/pge/util/GenericPgeObjectFactory.java
index d6e31dd..272c657 100644
--- a/pge/src/main/java/org/apache/oodt/cas/pge/util/GenericPgeObjectFactory.java
+++ b/pge/src/main/java/org/apache/oodt/cas/pge/util/GenericPgeObjectFactory.java
@@ -16,18 +16,12 @@
*/
package org.apache.oodt.cas.pge.util;
-
-//OODT imports
import org.apache.oodt.cas.pge.ConfigFilePropertyAdder;
import org.apache.oodt.cas.pge.PGETaskInstance;
import org.apache.oodt.cas.pge.config.PgeConfigBuilder;
import org.apache.oodt.cas.pge.staging.FileStager;
import org.apache.oodt.cas.pge.writers.SciPgeConfigFileWriter;
-//JDK imports
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
/**
* Factory for creating {@link Object}s.
@@ -38,66 +32,48 @@
private GenericPgeObjectFactory() {}
- public static PGETaskInstance createPGETaskInstance(
- String clazz, Logger logger) {
+ public static PGETaskInstance createPGETaskInstance(String clazz, org.slf4j.Logger logger) {
try {
return (PGETaskInstance) Class.forName(clazz).newInstance();
} catch (Exception e) {
- logger.log(Level.SEVERE, "Failed to create PGETaskInstance ["
- + clazz + "] : " + e.getMessage(), e);
+ logger.error("Failed to create PGETaskInstance [{}]", clazz, e);
return null;
}
}
- public static PgeConfigBuilder createPgeConfigBuilder(
- String clazz, Logger logger) {
+ public static PgeConfigBuilder createPgeConfigBuilder(String clazz, org.slf4j.Logger logger) {
try {
return (PgeConfigBuilder) Class.forName(clazz).newInstance();
- } catch (InstantiationException e) {
- logger.log(Level.SEVERE, "Failed to create PgeConfigBuilder ["
- + clazz + "] : " + e.getMessage(), e);
- return null;
- } catch (IllegalAccessException e) {
- logger.log(Level.SEVERE, "Failed to create PgeConfigBuilder ["
- + clazz + "] : " + e.getMessage(), e);
- return null;
- } catch (ClassNotFoundException e) {
- logger.log(Level.SEVERE, "Failed to create PgeConfigBuilder ["
- + clazz + "] : " + e.getMessage(), e);
+ } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+ logger.error("Failed to create PgeConfigBuilder [{}] : ", clazz, e);
return null;
}
}
- public static ConfigFilePropertyAdder createConfigFilePropertyAdder(
- String clazz, Logger logger) {
+ public static ConfigFilePropertyAdder createConfigFilePropertyAdder(String clazz, org.slf4j.Logger logger) {
try {
return (ConfigFilePropertyAdder) Class.forName(clazz).newInstance();
} catch (Exception e) {
- logger.log(Level.SEVERE, "Failed to create ConfigFilePropertyAdder ["
- + clazz + "] : " + e.getMessage(), e);
+ logger.error("Failed to create ConfigFilePropertyAdder [{}]", clazz, e);
return null;
}
}
- public static FileStager createFileStager(
- String clazz, Logger logger) {
+ public static FileStager createFileStager(String clazz, org.slf4j.Logger logger) {
try {
return (FileStager) Class.forName(clazz).newInstance();
} catch (Exception e) {
- logger.log(Level.SEVERE, "Failed to create FileStager ["
- + clazz + "] : " + e.getMessage(), e);
+ logger.error("Failed to create FileStager [{}]", clazz, e);
return null;
}
}
- public static SciPgeConfigFileWriter createSciPgeConfigFileWriter(
- String clazz, Logger logger) {
+ public static SciPgeConfigFileWriter createSciPgeConfigFileWriter(String clazz, org.slf4j.Logger logger) {
try {
return (SciPgeConfigFileWriter) Class.forName(clazz).newInstance();
} catch (Exception e) {
- logger.log(Level.SEVERE, "Failed to create SciPgeConfigFileWriter ["
- + clazz + "] : " + e.getMessage(), e);
+ logger.error("Failed to create SciPgeConfigFileWriter [{}]", clazz, e);
return null;
}
}
diff --git a/pge/src/test/java/org/apache/oodt/cas/pge/TestPGETaskInstance.java b/pge/src/test/java/org/apache/oodt/cas/pge/TestPGETaskInstance.java
index db60792..2378849 100644
--- a/pge/src/test/java/org/apache/oodt/cas/pge/TestPGETaskInstance.java
+++ b/pge/src/test/java/org/apache/oodt/cas/pge/TestPGETaskInstance.java
@@ -189,15 +189,15 @@
PGETaskInstance pgeTask1 = createTestInstance();
PGETaskInstance pgeTask2 = createTestInstance();
- pgeTask1.logger.log(Level.INFO, "pge1 message1");
- pgeTask1.logger.log(Level.INFO, "pge1 message2");
- pgeTask2.logger.log(Level.SEVERE, "pge2 message1");
- pgeTask1.logger.log(Level.INFO, "pge1 message3");
+ pgeTask1.julLogger.log(Level.INFO, "pge1 message1");
+ pgeTask1.julLogger.log(Level.INFO, "pge1 message2");
+ pgeTask2.julLogger.log(Level.SEVERE, "pge2 message1");
+ pgeTask1.julLogger.log(Level.INFO, "pge1 message3");
- for (Handler handler : pgeTask1.logger.getHandlers()) {
+ for (Handler handler : pgeTask1.julLogger.getHandlers()) {
handler.flush();
}
- for (Handler handler : pgeTask2.logger.getHandlers()) {
+ for (Handler handler : pgeTask2.julLogger.getHandlers()) {
handler.flush();
}
File logDir = new File(pgeTask1.pgeConfig.getExeDir() + "/logs");
@@ -586,17 +586,17 @@
Lists.newArrayList(precondsFailIngestStatus));
replay(pc);
- pgeTask.logger = createMock(Logger.class);
- pgeTask.logger.info("Verifying ingests successful...");
- pgeTask.logger.warning(
+ pgeTask.julLogger = createMock(Logger.class);
+ pgeTask.julLogger.info("Verifying ingests successful...");
+ pgeTask.julLogger.warning(
"Product was not ingested [file='/tmp/dir1',result='PRECONDS_FAILED',msg='Preconditions failed']");
- pgeTask.logger.info("Ingests were successful");
- replay(pgeTask.logger);
+ pgeTask.julLogger.info("Ingests were successful");
+ replay(pgeTask.julLogger);
pgeTask.verifyIngests(pc);
verify(pc);
- verify(pgeTask.logger);
+ verify(pgeTask.julLogger);
// Test case success.
pc = createMock(AutoDetectProductCrawler.class);
@@ -618,15 +618,15 @@
Lists.newArrayList(successIngestStatus));
replay(pc);
- pgeTask.logger = createMock(Logger.class);
- pgeTask.logger.info("Verifying ingests successful...");
- pgeTask.logger.info("Ingests were successful");
- replay(pgeTask.logger);
+ pgeTask.julLogger = createMock(Logger.class);
+ pgeTask.julLogger.info("Verifying ingests successful...");
+ pgeTask.julLogger.info("Ingests were successful");
+ replay(pgeTask.julLogger);
pgeTask.verifyIngests(pc);
verify(pc);
- verify(pgeTask.logger);
+ verify(pgeTask.julLogger);
}
private PGETaskInstance createTestInstance() throws Exception {
@@ -642,7 +642,7 @@
pgeTask.pgeConfig = new PgeConfig();
File exeDir = new File(createTmpDir(), workflowInstId);
pgeTask.pgeConfig.setExeDir(exeDir.getAbsolutePath());
- pgeTask.logger = pgeTask.createLogger();
+ pgeTask.julLogger = pgeTask.createLogger();
return pgeTask;
}
