TENTACLES-8 NexusClient not closing connections
* Applying patch from Andy Gumprecht to raise number of connections.
* Reformatted, added constants.
* Removed unused dependency to httpcore.
* Remove resource-leakage-warning by extracting encapsulated inputStream.
git-svn-id: https://svn.apache.org/repos/asf/creadur/tentacles/trunk@1624903 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/pom.xml b/pom.xml
index 1833f3f..2078aac 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,11 +45,6 @@
<version>${httpClientVersion}</version>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- <version>${httpClientVersion}</version>
- </dependency>
- <dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-jaxb-annotations</artifactId>
<version>2.4.2</version>
@@ -79,7 +74,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<javaVersion>1.6</javaVersion>
- <httpClientVersion>4.3.2</httpClientVersion>
+ <httpClientVersion>4.3.5</httpClientVersion>
<apacheRatVersion>0.11</apacheRatVersion>
</properties>
<issueManagement>
@@ -111,7 +106,7 @@
</mailingList>
</mailingLists>
<build>
- <defaultGoal>install</defaultGoal>
+ <defaultGoal>clean install</defaultGoal>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/src/main/java/org/apache/creadur/tentacles/Deauthorize.java b/src/main/java/org/apache/creadur/tentacles/Deauthorize.java
index 6f72b41..0ed882d 100644
--- a/src/main/java/org/apache/creadur/tentacles/Deauthorize.java
+++ b/src/main/java/org/apache/creadur/tentacles/Deauthorize.java
@@ -34,135 +34,134 @@
*/
public class Deauthorize {
- /**
- * All input must be valid directories.
- *
- * Invalid input is logged to System.err and skipped
- *
- * @param args
- * a list of directories to scan and fix
- * @throws Exception in case of errors.
- */
- public static void main(final String[] args) throws Exception {
+ /**
+ * All input must be valid directories.
+ *
+ * Invalid input is logged to System.err and skipped
+ *
+ * @param args
+ * a list of directories to scan and fix
+ * @throws Exception
+ * in case of errors.
+ */
+ public static void main(final String[] args) throws Exception {
- if (args.length == 0) {
- throw new IllegalArgumentException(
- "At least one directory must be specified");
- }
+ if (args.length == 0) {
+ throw new IllegalArgumentException(
+ "At least one directory must be specified");
+ }
- final List<File> dirs = new ArrayList<File>();
+ final List<File> dirs = new ArrayList<File>();
- // Check the input args upfront
- for (final String arg : args) {
- final File dir = new File(arg);
+ // Check the input args upfront
+ for (final String arg : args) {
+ final File dir = new File(arg);
- if (not(dir.exists(), "Does not exist: %s", arg)) {
- continue;
- }
- if (not(dir.isDirectory(), "Not a directory: %s", arg)) {
- continue;
- }
+ if (not(dir.exists(), "Does not exist: %s", arg)) {
+ continue;
+ }
+ if (not(dir.isDirectory(), "Not a directory: %s", arg)) {
+ continue;
+ }
- dirs.add(dir);
- }
+ dirs.add(dir);
+ }
- // Exit if we got bad input
- if (dirs.size() != args.length) {
- System.exit(1);
- }
+ // Exit if we got bad input
+ if (dirs.size() != args.length) {
+ System.exit(1);
+ }
- // Go!
- for (final File dir : dirs) {
- deauthorize(dir);
- }
- }
+ // Go!
+ for (final File dir : dirs) {
+ deauthorize(dir);
+ }
+ }
- /**
- * Iterate over all the java files in the given directory
- *
- * Read in the file so we can guess the line ending -- if we didn't need to
- * do that we could just stream. Run the content through Swizzle Stream and
- * filter out any author tags as well as any comment blocks that wind up (or
- * already were) empty as a result.
- *
- * If that had any effect on the contents of the file, write it back out.
- *
- * Should skip any files that are not readable or writable.
- *
- * Will log an error on System.err for any files that were updated and were
- * not writable. Files that are not writable and don't need updating are
- * simply ignored.
- *
- * @param dir
- * @throws IOException
- */
- private static void deauthorize(final File dir) throws IOException {
- deauthorize(dir, new IOSystem());
- }
+ /**
+ * Iterate over all the java files in the given directory
+ *
+ * Read in the file so we can guess the line ending -- if we didn't need to
+ * do that we could just stream. Run the content through Swizzle Stream and
+ * filter out any author tags as well as any comment blocks that wind up (or
+ * already were) empty as a result.
+ *
+ * If that had any effect on the contents of the file, write it back out.
+ *
+ * Should skip any files that are not readable or writable.
+ *
+ * Will log an error on System.err for any files that were updated and were
+ * not writable. Files that are not writable and don't need updating are
+ * simply ignored.
+ *
+ * @param dir
+ * @throws IOException
+ */
+ private static void deauthorize(final File dir) throws IOException {
+ deauthorize(dir, new IOSystem());
+ }
- private static void deauthorize(final File dir, final IOSystem io)
- throws IOException {
- for (final File file : new FileSystem().collect(dir, ".*\\.java")) {
+ private static void deauthorize(final File dir, final IOSystem io)
+ throws IOException {
+ for (final File file : new FileSystem().collect(dir, ".*\\.java")) {
- if (not(file.canRead(), "File not readable: %s",
- file.getAbsolutePath())) {
- continue;
- }
+ if (not(file.canRead(), "File not readable: %s",
+ file.getAbsolutePath())) {
+ continue;
+ }
- final String text = io.slurp(file);
+ final String text = io.slurp(file);
- // You really can't trust text to be in the native line ending
- final String eol = text.contains("\r\n") ? "\r\n" : "\n";
- final String startComment = eol + "/*";
- final String endComment = "*/" + eol;
+ // You really can't trust text to be in the native line ending
+ final String eol = text.contains("\r\n") ? "\r\n" : "\n";
+ final String startComment = eol + "/*";
+ final String endComment = "*/" + eol;
- InputStream in = new ByteArrayInputStream(text.getBytes());
+ final InputStream baseIn = new ByteArrayInputStream(text.getBytes());
- // Yank author tags
- in = new ExcludeFilterInputStream(in, " * @author", eol);
+ // Yank author tags
+ InputStream in = new ExcludeFilterInputStream(baseIn, " * @author",
+ eol);
- // Clean "empty" comments
- in =
- new DelimitedTokenReplacementInputStream(in, startComment,
- endComment, new StringTokenHandler() {
- @Override
- public String handleToken(
- final String commentBlock)
- throws IOException {
+ // Clean "empty" comments
+ in = new DelimitedTokenReplacementInputStream(in, startComment,
+ endComment, new StringTokenHandler() {
+ @Override
+ public String handleToken(final String commentBlock)
+ throws IOException {
- // Yank if empty
- if (commentBlock.replaceAll("[\\s*]", "")
- .length() == 0) {
- return eol;
- }
+ // Yank if empty
+ if (commentBlock.replaceAll("[\\s*]", "").length() == 0) {
+ return eol;
+ }
- // Keep otherwise
- return startComment + commentBlock
- + endComment;
- }
- });
+ // Keep otherwise
+ return startComment + commentBlock + endComment;
+ }
+ });
- final byte[] content = io.read(in);
+ final byte[] content = io.read(in);
- if (content.length != file.length()) {
+ if (content.length != file.length()) {
- if (not(file.canWrite(), "File not writable: %s",
- file.getAbsolutePath())) {
- continue;
- }
+ if (not(file.canWrite(), "File not writable: %s",
+ file.getAbsolutePath())) {
+ continue;
+ }
- io.copy(content, file);
- }
- }
- }
+ io.copy(content, file);
+ }
+ }
- private static boolean not(boolean b, final String message,
- final Object... details) {
- b = !b;
- if (b) {
- System.err.printf(message, details);
- System.err.println();
- }
- return b;
- }
+ }
+
+ private static boolean not(boolean b, final String message,
+ final Object... details) {
+ b = !b;
+ if (b) {
+ System.err.printf(message, details);
+ System.err.println();
+ }
+ return b;
+ }
}
diff --git a/src/main/java/org/apache/creadur/tentacles/NexusClient.java b/src/main/java/org/apache/creadur/tentacles/NexusClient.java
index b9fbd64..8f793de 100644
--- a/src/main/java/org/apache/creadur/tentacles/NexusClient.java
+++ b/src/main/java/org/apache/creadur/tentacles/NexusClient.java
@@ -24,126 +24,144 @@
import java.util.Set;
import org.apache.http.Header;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
+import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.log4j.Logger;
import org.codehaus.swizzle.stream.StreamLexer;
public class NexusClient {
- private static final org.apache.log4j.Logger log = org.apache.log4j.Logger
- .getLogger(NexusClient.class);
+ private static final Logger log = Logger.getLogger(NexusClient.class);
+ private static final String SLASH = "/";
+ private static final String ONE_UP = "../";
+ private static final String USER_AGENT_CONTENTS = "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101206 Ubuntu/10.10 (maverick) Firefox/3.6.13";
- private final HttpClient client;
- private final FileSystem fileSystem;
- private final IOSystem ioSystem;
+ private final CloseableHttpClient client;
+ private final FileSystem fileSystem;
+ private final IOSystem ioSystem;
- public NexusClient(final Platform platform) {
- this.client = HttpClientBuilder.create().build();
- this.fileSystem = platform.getFileSystem();
- this.ioSystem = platform.getIoSystem();
- }
+ public NexusClient(final Platform platform) {
- public File download(final URI uri, final File file) throws IOException {
- if (file.exists()) {
+ System.setProperty("http.keepAlive", "false");
+ System.setProperty("http.maxConnections", "50");
- final long length = getContentLength(uri);
+ this.client = HttpClientBuilder.create().disableContentCompression()
+ .build();
+ this.fileSystem = platform.getFileSystem();
+ this.ioSystem = platform.getIoSystem();
+ }
- if (file.length() == length) {
- log.info("Exists " + uri);
- return file;
- } else {
- log.info("Incomplete " + uri);
- }
- }
+ public File download(final URI uri, final File file) throws IOException {
+ if (file.exists()) {
- log.info("Download " + uri);
+ final long length = getContentLength(uri);
- final HttpResponse response = get(uri);
+ if (file.length() == length) {
+ log.info("Exists " + uri);
+ return file;
+ } else {
+ log.info("Incomplete " + uri);
+ }
+ }
- final InputStream content = response.getEntity().getContent();
+ log.info("Download " + uri);
- this.fileSystem.mkparent(file);
+ final CloseableHttpResponse response = get(uri);
- this.ioSystem.copy(content, file);
+ InputStream content = null;
+ try {
+ content = response.getEntity().getContent();
- return file;
- }
+ this.fileSystem.mkparent(file);
- private long getContentLength(final URI uri) throws IOException {
- final HttpResponse head = head(uri);
- final Header[] headers = head.getHeaders("Content-Length");
+ this.ioSystem.copy(content, file);
+ } finally {
+ if (content != null) {
+ content.close();
+ }
- if(headers != null && headers.length >= 1) {
- return Long.valueOf(headers[0].getValue());
- }
- return -1;
- }
+ response.close();
+ }
- private HttpResponse get(final URI uri) throws IOException {
- final HttpGet request = new HttpGet(uri);
- request.setHeader(
- "User-Agent",
- "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101206 Ubuntu/10.10 (maverick) Firefox/3.6.13");
- return this.client.execute(request);
- }
+ return file;
+ }
- private HttpResponse head(final URI uri) throws IOException {
- final HttpHead request = new HttpHead(uri);
- request.setHeader(
- "User-Agent",
- "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101206 Ubuntu/10.10 (maverick) Firefox/3.6.13");
- return this.client.execute(request);
- }
+ private Long getContentLength(final URI uri) throws IOException {
+ final CloseableHttpResponse head = head(uri);
+ final Header[] headers = head.getHeaders(HttpHeaders.CONTENT_LENGTH);
- public Set<URI> crawl(final URI index) throws IOException {
- log.info("Crawl " + index);
- final Set<URI> resources = new LinkedHashSet<URI>();
+ if (headers != null && headers.length >= 1) {
+ return Long.valueOf(headers[0].getValue());
+ }
- final HttpResponse response = get(index);
+ head.close();
- final InputStream content = response.getEntity().getContent();
- final StreamLexer lexer = new StreamLexer(content);
+ return Long.valueOf(-1);
+ }
- final Set<URI> crawl = new LinkedHashSet<URI>();
+ private CloseableHttpResponse get(final URI uri) throws IOException {
+ final HttpGet request = new HttpGet(uri);
+ request.setHeader(HttpHeaders.USER_AGENT, USER_AGENT_CONTENTS);
+ return this.client.execute(request);
+ }
- // <a
- // href="https://repository.apache.org/content/repositories/orgapacheopenejb-094/archetype-catalog.xml">archetype-catalog.xml</a>
- while (lexer.readAndMark("<a ", "/a>")) {
+ private CloseableHttpResponse head(final URI uri) throws IOException {
+ final HttpHead request = new HttpHead(uri);
+ request.setHeader(HttpHeaders.USER_AGENT, USER_AGENT_CONTENTS);
+ return this.client.execute(request);
+ }
- try {
- final String link = lexer.peek("href=\"", "\"");
- final String name = lexer.peek(">", "<");
+ public Set<URI> crawl(final URI index) throws IOException {
+ log.info("Crawl " + index);
+ final Set<URI> resources = new LinkedHashSet<URI>();
- final URI uri = index.resolve(link);
+ final CloseableHttpResponse response = get(index);
- if (name.equals("../")) {
- continue;
- }
- if (link.equals("../")) {
- continue;
- }
+ final InputStream content = response.getEntity().getContent();
+ final StreamLexer lexer = new StreamLexer(content);
- if (name.endsWith("/")) {
- crawl.add(uri);
- continue;
- }
+ final Set<URI> crawl = new LinkedHashSet<URI>();
- resources.add(uri);
+ // <a
+ // href="https://repository.apache.org/content/repositories/orgapacheopenejb-094/archetype-catalog.xml">archetype-catalog.xml</a>
+ while (lexer.readAndMark("<a ", "/a>")) {
- } finally {
- lexer.unmark();
- }
- }
+ try {
+ final String link = lexer.peek("href=\"", "\"");
+ final String name = lexer.peek(">", "<");
- content.close();
+ final URI uri = index.resolve(link);
- for (final URI uri : crawl) {
- resources.addAll(crawl(uri));
- }
+ if (name.equals(ONE_UP)) {
+ continue;
+ }
+ if (link.equals(ONE_UP)) {
+ continue;
+ }
- return resources;
- }
+ if (name.endsWith(SLASH)) {
+ crawl.add(uri);
+ continue;
+ }
+
+ resources.add(uri);
+
+ } finally {
+ lexer.unmark();
+ }
+ }
+
+ content.close();
+ response.close();
+
+ for (final URI uri : crawl) {
+ resources.addAll(crawl(uri));
+ }
+
+ return resources;
+ }
}
\ No newline at end of file
