[OODT-978] Fixed crawler launcher hanging issue in file manager avro version
by adding a close() method to the FileManagerClient
diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java
index 9fd682c..b47c7e2 100644
--- a/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java
+++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java
@@ -18,13 +18,14 @@
//JDK imports
import java.io.IOException;
+import java.util.Set;
//OODT imports
import org.apache.oodt.cas.cli.CmdLineUtility;
/**
* A command line interface to the new Spring enabled crawler.
- *
+ *
* @author bfoster (Brian Foster)
* @version $Revision$
* @since OODT-190
@@ -34,5 +35,6 @@
public static void main(String[] args) throws IOException {
CmdLineUtility cmdLineUtility = new CmdLineUtility();
cmdLineUtility.run(args);
+ System.out.println("Exiting crawler launcher");
}
}
diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
index 3f5d254..494b2c2 100644
--- a/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
+++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
@@ -17,8 +17,6 @@
package org.apache.oodt.cas.crawl;
import com.google.common.annotations.VisibleForTesting;
-
-//OODT imports
import org.apache.oodt.cas.crawl.action.CrawlerAction;
import org.apache.oodt.cas.crawl.action.CrawlerActionRepo;
import org.apache.oodt.cas.crawl.config.ProductCrawlerBean;
@@ -29,6 +27,7 @@
import java.io.File;
import java.io.FileFilter;
+import java.io.IOException;
import java.net.URL;
import java.util.Collections;
import java.util.List;
@@ -37,6 +36,8 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+//OODT imports
+
/**
* An abstract base class for Product Crawling. This class provides methods to
@@ -71,11 +72,15 @@
protected Ingester ingester;
public void crawl() {
+ LOG.info("Crawl operation invoked");
crawl(new File(getProductPath()));
}
public void crawl(File dirRoot) {
- // Reset ingest status.
+ LOG.info(String.format("Start crawling dir: %s", dirRoot));
+
+ // Reset ingest status.a
+
ingestStatus.clear();
// Load actions.
@@ -114,6 +119,8 @@
}
}
}
+
+ LOG.info(String.format("Finished crawling dir: %s", dirRoot));
}
public IngestStatus handleFile(File product) {
@@ -363,4 +370,10 @@
}
return allSucceeded;
}
+
+ public void shutdown() throws IOException {
+ if (ingester != null) {
+ ingester.close();
+ }
+ }
}
diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java
index cce8ea8..b7b70ef 100644
--- a/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java
+++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java
@@ -27,7 +27,7 @@
/**
* A {@link CmdLineAction} which is responsible for launching crawlers.
- *
+ *
* @author bfoster (Brian Foster)
*/
public class CrawlerLauncherCliAction extends CmdLineAction {
@@ -54,9 +54,12 @@
if (pc.getDaemonPort() != -1 && pc.getDaemonWait() != -1) {
new CrawlDaemon(pc.getDaemonWait(), pc, pc.getDaemonPort())
.startCrawling();
+ printer.println("Finished crawler daemon");
} else {
pc.crawl();
+ printer.println("Finished crawling");
}
+ pc.shutdown();
} catch (Exception e) {
throw new CmdLineActionException("Failed to launch crawler : "
+ e.getMessage(), e);
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java
index 8e96be1..52e5c9f 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java
@@ -25,6 +25,7 @@
import java.io.File;
import java.io.FileInputStream;
+import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.net.URL;
@@ -39,7 +40,7 @@
*
* <p>
* An extension of the {@link StdIngester} that uses a {@link Cache} to keep
- * track of {@link Product} ingestion status. If the existing {@link Cache} used
+ * track of {@link org.apache.oodt.cas.filemgr.structs.Product} ingestion status. If the existing {@link Cache} used
* is already sync'ed to the requested File Manager (specified by the
* <code>fmUrl</code> parameter in {@link #hasProduct(URL, File)} or
* {@link #hasProduct(URL, String)}), then the {@link Cache} will simply return
@@ -58,7 +59,7 @@
/**
* @param transferService
* The underlying data transfer service to use to ingest
- * {@link Product}s.
+ * {@link org.apache.oodt.cas.filemgr.structs.Product}s.
* @param cacheServiceFactory
* The {@link CacheFactory} to use to construct this
* {@link Ingester}'s {@link Cache}.
@@ -94,7 +95,7 @@
*
* @param transferService
* The underlying data transfer service to use to ingest
- * {@link Product}s.
+ * {@link org.apache.oodt.cas.filemgr.structs.Product}s.
* @param cache
* The {@link Cache} that this {@link Ingester} will use.
* @throws InstantiationException
@@ -167,4 +168,10 @@
+ e.getMessage());
}
}
+
+ @Override
+ public void close() throws IOException {
+ cache.clear();
+ super.close();
+ }
}
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java
index 59d0736..8d6807b 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java
@@ -128,6 +128,7 @@
System.out.println("Result: " + productID);
}
+ ingester.close();
}
private static List<String> readProdFilesFromStdin() {
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java
index b12dfd4..8e19273 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java
@@ -24,6 +24,7 @@
import org.apache.oodt.cas.metadata.Metadata;
//JDK imports
+import java.io.Closeable;
import java.io.File;
import java.net.URL;
import java.util.List;
@@ -37,7 +38,7 @@
* An interface for ingesting {@link Product}s
* </p>.
*/
-public interface Ingester {
+public interface Ingester extends Closeable {
/**
* Ingests a {@link Product} to the file manager service object identified
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java
index 2421873..0195bb6 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java
@@ -18,6 +18,7 @@
package org.apache.oodt.cas.filemgr.ingest;
//OODT imports
+
import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductType;
@@ -26,15 +27,15 @@
import org.apache.oodt.cas.filemgr.structs.exceptions.IngestException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
+import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.filemgr.versioning.VersioningUtils;
import org.apache.oodt.cas.metadata.MetExtractor;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.exceptions.MetExtractionException;
-//JDK imports
import java.io.File;
+import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.List;
@@ -42,26 +43,28 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+//JDK imports
+
/**
* @author mattmann
* @author bfoster
* @version $Revision$
- *
+ *
* <p>
* An implementation of the {@link Ingster} interface that uses the following
* pieces of {@link Metadata} information to determine how to ingest a
* {@link Product}:
- *
+ *
* <ul>
* <li>Filename - The name of the Product file to ingest.</li>
* <li>ProductType - The type of the Product to ingest.</li>
* <li>FileLocation - A full path pointer to directory containing the Product
* file to ingest.</li>
* </ul>
- *
+ *
* The {@link Ingester} supports overriding certain {@link Product} properties,
* including:
- *
+ *
* <ul>
* <li>Specification of <code>ProductStructure</code> parameter that will
* tell the {@link Ingester} whether or not the {@link Product} is a directory
@@ -90,7 +93,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.ingest.Ingester#ingest(java.net.URL,
* java.io.File, org.apache.oodt.cas.metadata.MetExtractor,
* java.io.File)
@@ -111,7 +114,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.ingest.Ingester#ingest(java.net.URL,
* java.util.List, org.apache.oodt.cas.metadata.MetExtractor, java.io.File)
*/
@@ -143,7 +146,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.ingest.Ingester#ingest(java.net.URL,
* java.io.File, org.apache.oodt.cas.metadata.Metadata)
*/
@@ -224,7 +227,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.ingest.Ingester#hasProduct(java.net.URL,
* java.io.File)
*/
@@ -293,7 +296,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.ingest.Ingester#hasProduct(java.net.URL,
* java.lang.String)
*/
@@ -309,4 +312,10 @@
}
}
+ @Override
+ public void close() throws IOException {
+ if (fmClient != null) {
+ fmClient.close();
+ }
+ }
}
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
index ca3839d..0b28bff 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
@@ -88,7 +88,7 @@
//setup the and start the client
try {
this.fileManagerUrl = url;
- InetSocketAddress inetSocketAddress = new InetSocketAddress(url.getHost(),this.fileManagerUrl.getPort());
+ InetSocketAddress inetSocketAddress = new InetSocketAddress(url.getHost(), this.fileManagerUrl.getPort());
this.client = new NettyTransceiver(inetSocketAddress, 40000L);
proxy = (AvroFileManager) SpecificRequestor.getClient(AvroFileManager.class, client);
} catch (IOException e) {
@@ -124,8 +124,7 @@
try {
if (proxy != null) {
success = proxy.isAlive();
- }
- else return false;
+ } else return false;
} catch (AvroRemoteException e) {
LOG.log(Level.WARNING, "AvroRemoteException when connecting to filemgr: ["
+ this.fileManagerUrl + "]");
@@ -155,7 +154,7 @@
success = proxy.removeProductTransferStatus(AvroTypeFactory.getAvroProduct(product));
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- }
+ }
return success;
}
@@ -584,7 +583,7 @@
@Override
public String ingestProduct(Product product, Metadata metadata,
- boolean clientTransfer) throws Exception {
+ boolean clientTransfer) throws Exception {
try {
// ingest product
String productId = this.proxy.ingestProduct(
@@ -594,7 +593,7 @@
if (clientTransfer) {
LOG.log(Level.FINEST, "File Manager Client: clientTransfer enabled: transfering product ["
- + product.getProductName() + "]");
+ + product.getProductName() + "]");
// we need to transfer the product ourselves
// make sure we have the product ID
@@ -736,4 +735,10 @@
this.dataTransfer = dataTransfer;
}
+ @Override
+ public void close() throws IOException {
+ if (client != null) {
+ client.close();
+ }
+ }
}
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java
index 8b1c54a..c0fa46a 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java
@@ -35,6 +35,8 @@
import org.apache.oodt.cas.metadata.Metadata;
//JDK imports
+import java.io.Closeable;
+import java.io.IOException;
import java.net.URL;
import java.util.List;
@@ -45,7 +47,7 @@
* <p>Interface of client for FileManager RPC logic. All methods that are used for
* RPC transfer.</p>
*/
-public interface FileManagerClient {
+public interface FileManagerClient extends Closeable {
public boolean refreshConfigAndPolicy();
@@ -176,4 +178,5 @@
public void setDataTransfer(DataTransfer dataTransfer);
+ void close() throws IOException;
}
\ No newline at end of file
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
index 3bd91fd..06c2726 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
@@ -1404,4 +1404,9 @@
this.dataTransfer.setFileManagerUrl(this.fileManagerUrl);
}
+ @Override
+ public void close() throws IOException {
+
+ }
+
}
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java
index 6a4c2bb..ebf42f0 100644
--- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java
@@ -285,4 +285,9 @@
public void setDataTransfer(DataTransfer dataTransfer) {
}
+
+ @Override
+ public void close() throws IOException {
+
+ }
}
