Refactored so some of the logic can be more easily reused
git-svn-id: https://svn.apache.org/repos/asf/incubator/rat/tentacles/trunk@1232283 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/src/main/java/org/apache/rat/tentacles/Files.java b/src/main/java/org/apache/rat/tentacles/Files.java
index d3a664c..a0661bd 100644
--- a/src/main/java/org/apache/rat/tentacles/Files.java
+++ b/src/main/java/org/apache/rat/tentacles/Files.java
@@ -77,4 +77,20 @@
if (file.exists()) return;
if (!file.mkdirs()) throw new RuntimeException("Cannot mkdir: " + file.getAbsolutePath());
}
+
+ public static void mkparent(File file) {
+ mkdirs(file.getParentFile());
+ }
+
+ public static void mkdirs(File file) {
+
+ if (!file.exists()) {
+
+ assert file.mkdirs() : "mkdirs " + file;
+
+ return;
+ }
+
+ assert file.isDirectory() : "not a directory" + file;
+ }
}
diff --git a/src/main/java/org/apache/rat/tentacles/Main.java b/src/main/java/org/apache/rat/tentacles/Main.java
index 9ff7ae1..5858c81 100644
--- a/src/main/java/org/apache/rat/tentacles/Main.java
+++ b/src/main/java/org/apache/rat/tentacles/Main.java
@@ -16,21 +16,14 @@
*/
package org.apache.rat.tentacles;
-import org.apache.http.Header;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpHead;
-import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
-import org.codehaus.swizzle.stream.StreamLexer;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
-import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@@ -38,6 +31,7 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -61,7 +55,6 @@
private static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(Main.class);
- private final DefaultHttpClient client;
private final File local;
private final URI staging;
private final File repository;
@@ -69,10 +62,10 @@
private Reports reports;
private Map<String, String> licenses = new HashMap<String, String>();
private String filter;
+ private final NexusClient client = new NexusClient();
public Main(String... args) throws Exception {
- client = new DefaultHttpClient();
this.staging = getURI(args[0]);
@@ -84,13 +77,13 @@
this.local = new File(name);
}
- mkdirs(local);
+ Files.mkdirs(local);
this.repository = new File(local, "repo");
this.content = new File(local, "content");
- mkdirs(repository);
- mkdirs(content);
+ Files.mkdirs(repository);
+ Files.mkdirs(content);
log.info("Repo: " + staging);
log.info("Local: " + local);
@@ -142,7 +135,7 @@
archives.add(archive);
}
- Templates.template("archives.vm")
+ Templates.template("legal/archives.vm")
.add("archives", archives)
.add("reports", reports)
.write(new File(local, "archives.html"));
@@ -156,7 +149,7 @@
private void reportLicenses(List<Archive> archives) throws IOException {
initLicenses(archives);
- Templates.template("licenses.vm")
+ Templates.template("legal/licenses.vm")
.add("licenses", getLicenses(archives))
.add("reports", reports)
.write(new File(local, "licenses.html"));
@@ -199,7 +192,7 @@
}
for (Archive archive : archives) {
- Templates.template("archive-licenses.vm")
+ Templates.template("legal/archive-licenses.vm")
.add("archive", archive)
.add("reports", reports)
.write(new File(local, reports.licenses(archive)));
@@ -270,7 +263,7 @@
}
- Templates.template("archive-notices.vm")
+ Templates.template("legal/archive-notices.vm")
.add("archive", archive)
.add("reports", reports)
.write(new File(local, reports.notices(archive)));
@@ -297,7 +290,7 @@
}
}
- Templates.template("notices.vm")
+ Templates.template("legal/notices.vm")
.add("notices", notices.values())
.add("reports", reports)
.write(new File(local, "notices.html"));
@@ -335,9 +328,10 @@
final Set<File> files = new HashSet<File>();
if (staging.toString().startsWith("http")) {
- final Set<URI> resources = crawl(staging);
+ final Set<URI> resources = client.crawl(staging);
for (URI uri : resources) {
+ if (!uri.getPath().matches(".*(war|jar|zip)")) continue;
files.add(download(uri));
}
} else if (staging.toString().startsWith("file:")) {
@@ -379,7 +373,7 @@
final File fileEntry = new File(contents, path);
- mkparent(fileEntry);
+ Files.mkparent(fileEntry);
// Open the output file
@@ -527,7 +521,7 @@
if (path.startsWith("content/")) path = path.substring("content/".length());
final File contents = new File(content, path + ".contents");
- mkdirs(contents);
+ Files.mkdirs(contents);
return contents;
}
@@ -535,29 +529,7 @@
final File file = getFile(uri);
- if (file.exists()) {
-
- long length = getConentLength(uri);
-
- if (file.length() == length) {
- log.info("Exists " + uri);
- return file;
- } else {
- log.info("Incomplete " + uri);
- }
- }
-
- log.info("Download " + uri);
-
- final HttpResponse response = get(uri);
-
- final InputStream content = response.getEntity().getContent();
-
- mkparent(file);
-
- IO.copy(content, file);
-
- return file;
+ return client.download(uri, file);
}
private File copy(File src) throws IOException {
@@ -567,7 +539,7 @@
log.info("Copy " + uri);
- mkparent(file);
+ Files.mkparent(file);
IO.copy(IO.read(src), file);
@@ -756,95 +728,11 @@
}
}
- private long getConentLength(URI uri) throws IOException {
- HttpResponse head = head(uri);
- Header[] headers = head.getHeaders("Content-Length");
-
- for (Header header : headers) {
- return new Long(header.getValue());
- }
-
- return -1;
- }
-
private File getFile(URI uri) {
final String name = uri.toString().replace(staging.toString(), "").replaceFirst("^/", "");
return new File(repository, name);
}
- private void mkparent(File file) {
- mkdirs(file.getParentFile());
- }
-
- private void mkdirs(File file) {
-
- if (!file.exists()) {
-
- assert file.mkdirs() : "mkdirs " + file;
-
- return;
- }
-
- assert file.isDirectory() : "not a directory" + file;
- }
-
- private HttpResponse get(URI uri) throws IOException {
- final HttpGet request = new HttpGet(uri);
- request.setHeader("User-Agent", "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101206 Ubuntu/10.10 (maverick) Firefox/3.6.13");
- return client.execute(request);
- }
-
- private HttpResponse head(URI uri) throws IOException {
- final HttpHead request = new HttpHead(uri);
- request.setHeader("User-Agent", "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101206 Ubuntu/10.10 (maverick) Firefox/3.6.13");
- return client.execute(request);
- }
-
- private Set<URI> crawl(URI index) throws IOException {
- log.info("Crawl " + index);
- final Set<URI> resources = new LinkedHashSet<URI>();
-
- final HttpResponse response = get(index);
-
- final InputStream content = response.getEntity().getContent();
- final StreamLexer lexer = new StreamLexer(content);
-
- final Set<URI> crawl = new LinkedHashSet<URI>();
-
- //<a href="https://repository.apache.org/content/repositories/orgapacheopenejb-094/archetype-catalog.xml">archetype-catalog.xml</a>
- while (lexer.readAndMark("<a ", "/a>")) {
-
- try {
- final String link = lexer.peek("href=\"", "\"");
- final String name = lexer.peek(">", "<");
-
- final URI uri = index.resolve(link);
-
- if (name.equals("../")) continue;
- if (link.equals("../")) continue;
-
- if (name.endsWith("/")) {
- crawl.add(uri);
- continue;
- }
-
- if (!isValidArchive(uri.getPath())) continue;
-
- resources.add(uri);
-
- } finally {
- lexer.unmark();
- }
- }
-
- content.close();
-
- for (URI uri : crawl) {
- resources.addAll(crawl(uri));
- }
- return resources;
- }
-
private boolean isValidArchive(String path) {
return path.matches(".*\\.(jar|zip|war|ear|tar.gz)");
}
diff --git a/src/main/java/org/apache/rat/tentacles/NexusClient.java b/src/main/java/org/apache/rat/tentacles/NexusClient.java
new file mode 100644
index 0000000..2ec3c40
--- /dev/null
+++ b/src/main/java/org/apache/rat/tentacles/NexusClient.java
@@ -0,0 +1,119 @@
+package org.apache.rat.tentacles;
+
+import org.apache.http.Header;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.codehaus.swizzle.stream.StreamLexer;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+public class NexusClient {
+
+ private static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(NexusClient.class);
+
+ private final DefaultHttpClient client;
+
+ public NexusClient() {
+ this.client = new DefaultHttpClient();
+ }
+
+ public File download(URI uri, File file) throws IOException {
+ if (file.exists()) {
+
+ long length = getConentLength(uri);
+
+ if (file.length() == length) {
+ log.info("Exists " + uri);
+ return file;
+ } else {
+ log.info("Incomplete " + uri);
+ }
+ }
+
+ log.info("Download " + uri);
+
+ final HttpResponse response = get(uri);
+
+ final InputStream content = response.getEntity().getContent();
+
+ Files.mkparent(file);
+
+ IO.copy(content, file);
+
+ return file;
+ }
+
+ private long getConentLength(URI uri) throws IOException {
+ HttpResponse head = head(uri);
+ Header[] headers = head.getHeaders("Content-Length");
+
+ for (Header header : headers) {
+ return new Long(header.getValue());
+ }
+
+ return -1;
+ }
+
+ private HttpResponse get(URI uri) throws IOException {
+ final HttpGet request = new HttpGet(uri);
+ request.setHeader("User-Agent", "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101206 Ubuntu/10.10 (maverick) Firefox/3.6.13");
+ return client.execute(request);
+ }
+
+ private HttpResponse head(URI uri) throws IOException {
+ final HttpHead request = new HttpHead(uri);
+ request.setHeader("User-Agent", "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101206 Ubuntu/10.10 (maverick) Firefox/3.6.13");
+ return client.execute(request);
+ }
+
+ public Set<URI> crawl(URI index) throws IOException {
+ log.info("Crawl " + index);
+ final Set<URI> resources = new LinkedHashSet<URI>();
+
+ final HttpResponse response = get(index);
+
+ final InputStream content = response.getEntity().getContent();
+ final StreamLexer lexer = new StreamLexer(content);
+
+ final Set<URI> crawl = new LinkedHashSet<URI>();
+
+ //<a href="https://repository.apache.org/content/repositories/orgapacheopenejb-094/archetype-catalog.xml">archetype-catalog.xml</a>
+ while (lexer.readAndMark("<a ", "/a>")) {
+
+ try {
+ final String link = lexer.peek("href=\"", "\"");
+ final String name = lexer.peek(">", "<");
+
+ final URI uri = index.resolve(link);
+
+ if (name.equals("../")) continue;
+ if (link.equals("../")) continue;
+
+ if (name.endsWith("/")) {
+ crawl.add(uri);
+ continue;
+ }
+
+ resources.add(uri);
+
+ } finally {
+ lexer.unmark();
+ }
+ }
+
+ content.close();
+
+ for (URI uri : crawl) {
+ resources.addAll(crawl(uri));
+ }
+
+ return resources;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/rat/tentacles/Templates.java b/src/main/java/org/apache/rat/tentacles/Templates.java
index f8ff968..10365a3 100644
--- a/src/main/java/org/apache/rat/tentacles/Templates.java
+++ b/src/main/java/org/apache/rat/tentacles/Templates.java
@@ -16,13 +16,11 @@
*/
package org.apache.rat.tentacles;
-import org.apache.log4j.Logger;
import org.apache.velocity.VelocityContext;
import org.apache.velocity.app.VelocityEngine;
import org.apache.velocity.runtime.log.CommonsLogLogChute;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringWriter;
@@ -33,113 +31,36 @@
import java.util.Properties;
public final class Templates {
- private static final Logger LOGGER = Logger.getLogger(Templates.class);
private static final Templates INSTANCE = new Templates();
- private static final String LOG_TAG = Templates.class.getName();
- private static final String BASE = "legal/";
- private VelocityEngine engine;
- private Map<String, URL> resources = new HashMap<String, URL>();
+ private final VelocityEngine engine;
private Templates() {
- // no-op
- }
-
- public synchronized void init() {
- if (engine != null) {
- return;
- }
-
- engine = new VelocityEngine();
-
Properties properties = new Properties();
properties.setProperty("file.resource.loader.cache", "true");
properties.setProperty("resource.loader", "file, class");
properties.setProperty("class.resource.loader.description", "Velocity Classpath Resource Loader");
properties.setProperty("class.resource.loader.class", "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader");
properties.setProperty("runtime.log.logsystem.class", CommonsLogLogChute.class.getName());
- properties.setProperty("runtime.log.logsystem.commons.logging.name", LOG_TAG);
+ properties.setProperty("runtime.log.logsystem.commons.logging.name", Templates.class.getName());
+
+ engine = new VelocityEngine();
engine.init(properties);
}
private void evaluate(String template, Map<String, Object> mapContext, Writer writer) throws IOException {
- if (engine == null) {
- init();
- }
- if (!resources.containsKey(template)) {
- URL resource = Thread.currentThread().getContextClassLoader().getResource(BASE + template);
- resources.put(template, resource);
- }
+ URL resource = Thread.currentThread().getContextClassLoader().getResource(template);
- URL url = resources.get(template);
- if (url == null) {
- LOGGER.error("can't find template " + template);
- return;
- }
+ if (resource == null) throw new IllegalStateException(template);
VelocityContext context = new VelocityContext(mapContext);
- engine.evaluate(context, writer, LOG_TAG, new InputStreamReader(url.openStream()));
- }
-
- /**
- * generate a file from a velocity template.
- * <p/>
- * In error case (template not found...), only log with error level will be done (no exception).
- *
- * @param template the template path in PREFIX resource folder
- * @param mapContext the parameters of the template
- * @param path the output path
- */
- public void apply(String template, Map<String, Object> mapContext, String path) {
- FileWriter writer = null;
- try {
- writer = new FileWriter(path);
- evaluate(template, mapContext, writer);
- } catch (IOException ioe) {
- LOGGER.error("can't apply template " + template, ioe);
- } finally {
- if (writer != null) {
- try {
- writer.flush();
- writer.close();
- } catch (IOException e) {
- LOGGER.error("can't flush file " + path, e);
- }
- }
- }
- }
-
- public String apply(String template, Map<String, Object> mapContext) {
- StringWriter writer = null;
- try {
- writer = new StringWriter();
- evaluate(template, mapContext, writer);
- } catch (IOException ioe) {
- LOGGER.error("can't apply template " + template, ioe);
- } finally {
- if (writer != null) {
- try {
- writer.flush();
- writer.close();
- } catch (IOException e) {
- LOGGER.error("can't flush writer", e);
- }
- }
- }
- if (writer == null) {
- return "";
- }
- return writer.toString();
- }
-
- public static Templates get() {
- return INSTANCE;
+ engine.evaluate(context, writer, Templates.class.getName(), new InputStreamReader(resource.openStream()));
}
public static Builder template(String name) {
- return get().new Builder(name);
+ return INSTANCE.new Builder(name);
}
public class Builder {
@@ -161,7 +82,15 @@
}
public String apply() {
- return Templates.this.apply(template, map);
+ final StringWriter writer = new StringWriter();
+
+ try {
+ evaluate(template, map, writer);
+ } catch (IOException ioe) {
+ throw new RuntimeException("can't apply template " + template, ioe);
+ }
+
+ return writer.toString();
}
public File write(File file) throws IOException {
