| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.solr.util; |
| |
| import javax.xml.parsers.DocumentBuilderFactory; |
| import javax.xml.parsers.ParserConfigurationException; |
| import javax.xml.xpath.XPath; |
| import javax.xml.xpath.XPathConstants; |
| import javax.xml.xpath.XPathExpression; |
| import javax.xml.xpath.XPathExpressionException; |
| import javax.xml.xpath.XPathFactory; |
| import java.io.BufferedReader; |
| import java.io.ByteArrayInputStream; |
| import java.io.ByteArrayOutputStream; |
| import java.io.File; |
| import java.io.FileFilter; |
| import java.io.FileInputStream; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.InputStreamReader; |
| import java.io.OutputStream; |
| import java.net.HttpURLConnection; |
| import java.net.MalformedURLException; |
| import java.net.ProtocolException; |
| import java.net.URL; |
| import java.net.URLEncoder; |
| import java.nio.BufferOverflowException; |
| import java.nio.ByteBuffer; |
| import java.nio.charset.Charset; |
| import java.nio.charset.StandardCharsets; |
| import java.nio.file.InvalidPathException; |
| import java.security.GeneralSecurityException; |
| import java.text.SimpleDateFormat; |
| import java.util.ArrayList; |
| import java.util.Base64; |
| import java.util.Date; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.LinkedHashSet; |
| import java.util.List; |
| import java.util.Locale; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.TimeZone; |
| import java.util.regex.Pattern; |
| import java.util.regex.PatternSyntaxException; |
| import java.util.zip.GZIPInputStream; |
| import java.util.zip.Inflater; |
| import java.util.zip.InflaterInputStream; |
| |
| import org.w3c.dom.Document; |
| import org.w3c.dom.Node; |
| import org.w3c.dom.NodeList; |
| import org.xml.sax.SAXException; |
| |
| import static java.nio.charset.StandardCharsets.US_ASCII; |
| import static java.nio.charset.StandardCharsets.UTF_8; |
| |
| /** |
| * A simple utility class for posting raw updates to a Solr server, |
| * has a main method so it can be run on the command line. |
| * View this not as a best-practice code example, but as a standalone |
| * example built with an explicit purpose of not having external |
| * jar dependencies. |
| */ |
| public class SimplePostTool { |
| private static final String DEFAULT_POST_HOST = "localhost"; |
| private static final String DEFAULT_POST_PORT = "8983"; |
| private static final String VERSION_OF_THIS_TOOL = "5.0.0"; // TODO: hardcoded for now, but eventually to sync with actual Solr version |
| |
| private static final String DEFAULT_COMMIT = "yes"; |
| private static final String DEFAULT_OPTIMIZE = "no"; |
| private static final String DEFAULT_OUT = "no"; |
| private static final String DEFAULT_AUTO = "no"; |
| private static final String DEFAULT_RECURSIVE = "0"; |
| private static final int DEFAULT_WEB_DELAY = 10; |
| private static final int MAX_WEB_DEPTH = 10; |
| private static final String DEFAULT_CONTENT_TYPE = "application/xml"; |
| private static final String DEFAULT_FILE_TYPES = "xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log"; |
| private static final String BASIC_AUTH = "basicauth"; |
| |
| static final String DATA_MODE_FILES = "files"; |
| static final String DATA_MODE_ARGS = "args"; |
| static final String DATA_MODE_STDIN = "stdin"; |
| static final String DATA_MODE_WEB = "web"; |
| static final String DEFAULT_DATA_MODE = DATA_MODE_FILES; |
| |
| // Input args |
| boolean auto = false; |
| int recursive = 0; |
| int delay = 0; |
| String fileTypes; |
| URL solrUrl; |
| OutputStream out = null; |
| String type; |
| String format; |
| String mode; |
| boolean commit; |
| boolean optimize; |
| String[] args; |
| |
| private int currentDepth; |
| |
| static HashMap<String,String> mimeMap; |
| FileFilter fileFilter; |
| // Backlog for crawling |
| List<LinkedHashSet<URL>> backlog = new ArrayList<>(); |
| Set<URL> visited = new HashSet<>(); |
| |
| static final Set<String> DATA_MODES = new HashSet<>(); |
| static final String USAGE_STRING_SHORT = |
| "Usage: java [SystemProperties] -jar post.jar [-h|-] [<file|folder|url|arg> [<file|folder|url|arg>...]]"; |
| |
| // Used in tests to avoid doing actual network traffic |
| boolean mockMode = false; |
| PageFetcher pageFetcher; |
| |
| static { |
| DATA_MODES.add(DATA_MODE_FILES); |
| DATA_MODES.add(DATA_MODE_ARGS); |
| DATA_MODES.add(DATA_MODE_STDIN); |
| DATA_MODES.add(DATA_MODE_WEB); |
| |
| mimeMap = new HashMap<>(); |
| mimeMap.put("xml", "application/xml"); |
| mimeMap.put("csv", "text/csv"); |
| mimeMap.put("json", "application/json"); |
| mimeMap.put("jsonl", "application/json"); |
| mimeMap.put("pdf", "application/pdf"); |
| mimeMap.put("rtf", "text/rtf"); |
| mimeMap.put("html", "text/html"); |
| mimeMap.put("htm", "text/html"); |
| mimeMap.put("doc", "application/msword"); |
| mimeMap.put("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"); |
| mimeMap.put("ppt", "application/vnd.ms-powerpoint"); |
| mimeMap.put("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"); |
| mimeMap.put("xls", "application/vnd.ms-excel"); |
| mimeMap.put("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); |
| mimeMap.put("odt", "application/vnd.oasis.opendocument.text"); |
| mimeMap.put("ott", "application/vnd.oasis.opendocument.text"); |
| mimeMap.put("odp", "application/vnd.oasis.opendocument.presentation"); |
| mimeMap.put("otp", "application/vnd.oasis.opendocument.presentation"); |
| mimeMap.put("ods", "application/vnd.oasis.opendocument.spreadsheet"); |
| mimeMap.put("ots", "application/vnd.oasis.opendocument.spreadsheet"); |
| mimeMap.put("txt", "text/plain"); |
| mimeMap.put("log", "text/plain"); |
| } |
| |
| /** |
| * See usage() for valid command line usage |
| * @param args the params on the command line |
| */ |
| public static void main(String[] args) { |
| info("SimplePostTool version " + VERSION_OF_THIS_TOOL); |
| if (0 < args.length && ("-help".equals(args[0]) || "--help".equals(args[0]) || "-h".equals(args[0]))) { |
| usage(); |
| } else { |
| final SimplePostTool t = parseArgsAndInit(args); |
| t.execute(); |
| } |
| } |
| |
| /** |
| * After initialization, call execute to start the post job. |
| * This method delegates to the correct mode method. |
| */ |
| public void execute() { |
| final RTimer timer = new RTimer(); |
| if (DATA_MODE_FILES.equals(mode) && args.length > 0) { |
| doFilesMode(); |
| } else if(DATA_MODE_ARGS.equals(mode) && args.length > 0) { |
| doArgsMode(); |
| } else if(DATA_MODE_WEB.equals(mode) && args.length > 0) { |
| doWebMode(); |
| } else if(DATA_MODE_STDIN.equals(mode)) { |
| doStdinMode(); |
| } else { |
| usageShort(); |
| return; |
| } |
| |
| if (commit) commit(); |
| if (optimize) optimize(); |
| displayTiming((long) timer.getTime()); |
| } |
| |
| /** |
| * Pretty prints the number of milliseconds taken to post the content to Solr |
| * @param millis the time in milliseconds |
| */ |
| private void displayTiming(long millis) { |
| SimpleDateFormat df = new SimpleDateFormat("H:mm:ss.SSS", Locale.getDefault()); |
| df.setTimeZone(TimeZone.getTimeZone("UTC")); |
| System.out.println("Time spent: "+df.format(new Date(millis))); |
| } |
| |
| /** |
| * Parses incoming arguments and system params and initializes the tool |
| * @param args the incoming cmd line args |
| * @return an instance of SimplePostTool |
| */ |
| protected static SimplePostTool parseArgsAndInit(String[] args) { |
| String urlStr = null; |
| try { |
| // Parse args |
| final String mode = System.getProperty("data", DEFAULT_DATA_MODE); |
| if (! DATA_MODES.contains(mode)) { |
| fatal("System Property 'data' is not valid for this tool: " + mode); |
| } |
| |
| String params = System.getProperty("params", ""); |
| |
| String host = System.getProperty("host", DEFAULT_POST_HOST); |
| String port = System.getProperty("port", DEFAULT_POST_PORT); |
| String core = System.getProperty("c"); |
| |
| urlStr = System.getProperty("url"); |
| |
| if (urlStr == null && core == null) { |
| fatal("Specifying either url or core/collection is mandatory.\n" + USAGE_STRING_SHORT); |
| } |
| |
| if(urlStr == null) { |
| urlStr = String.format(Locale.ROOT, "http://%s:%s/solr/%s/update", host, port, core); |
| } |
| urlStr = SimplePostTool.appendParam(urlStr, params); |
| URL url = new URL(urlStr); |
| String user = null; |
| if (url.getUserInfo() != null && url.getUserInfo().trim().length() > 0) { |
| user = url.getUserInfo().split(":")[0]; |
| } else if (System.getProperty(BASIC_AUTH) != null) { |
| user = System.getProperty(BASIC_AUTH).trim().split(":")[0]; |
| } |
| if (user != null) |
| info("Basic Authentication enabled, user=" + user); |
| |
| boolean auto = isOn(System.getProperty("auto", DEFAULT_AUTO)); |
| String type = System.getProperty("type"); |
| String format = System.getProperty("format"); |
| // Recursive |
| int recursive = 0; |
| String r = System.getProperty("recursive", DEFAULT_RECURSIVE); |
| try { |
| recursive = Integer.parseInt(r); |
| } catch(Exception e) { |
| if (isOn(r)) |
| recursive = DATA_MODE_WEB.equals(mode)?1:999; |
| } |
| // Delay |
| int delay = DATA_MODE_WEB.equals(mode) ? DEFAULT_WEB_DELAY : 0; |
| try { |
| delay = Integer.parseInt(System.getProperty("delay", ""+delay)); |
| } catch(Exception e) { } |
| OutputStream out = isOn(System.getProperty("out", DEFAULT_OUT)) ? System.out : null; |
| String fileTypes = System.getProperty("filetypes", DEFAULT_FILE_TYPES); |
| boolean commit = isOn(System.getProperty("commit",DEFAULT_COMMIT)); |
| boolean optimize = isOn(System.getProperty("optimize",DEFAULT_OPTIMIZE)); |
| |
| return new SimplePostTool(mode, url, auto, type, format, recursive, delay, fileTypes, out, commit, optimize, args); |
| } catch (MalformedURLException e) { |
| fatal("System Property 'url' is not a valid URL: " + urlStr); |
| return null; |
| } |
| } |
| |
| /** |
| * Constructor which takes in all mandatory input for the tool to work. |
| * Also see usage() for further explanation of the params. |
| * @param mode whether to post files, web pages, params or stdin |
| * @param url the Solr base Url to post to, should end with /update |
| * @param auto if true, we'll guess type and add resourcename/url |
| * @param type content-type of the data you are posting |
| * @param recursive number of levels for file/web mode, or 0 if one file only |
| * @param delay if recursive then delay will be the wait time between posts |
| * @param fileTypes a comma separated list of file-name endings to accept for file/web |
| * @param out an OutputStream to write output to, e.g. stdout to print to console |
| * @param commit if true, will commit at end of posting |
| * @param optimize if true, will optimize at end of posting |
| * @param args a String[] of arguments, varies between modes |
| */ |
| public SimplePostTool(String mode, URL url, boolean auto, String type, String format, |
| int recursive, int delay, String fileTypes, OutputStream out, |
| boolean commit, boolean optimize, String[] args) { |
| this.mode = mode; |
| this.solrUrl = url; |
| this.auto = auto; |
| this.type = type; |
| this.format = format; |
| this.recursive = recursive; |
| this.delay = delay; |
| this.fileTypes = fileTypes; |
| this.fileFilter = getFileFilterFromFileTypes(fileTypes); |
| this.out = out; |
| this.commit = commit; |
| this.optimize = optimize; |
| this.args = args; |
| pageFetcher = new PageFetcher(); |
| } |
| |
| public SimplePostTool() {} |
| |
| // |
| // Do some action depending on which mode we have |
| // |
| private void doFilesMode() { |
| currentDepth = 0; |
| // Skip posting files if special param "-" given |
| if (!args[0].equals("-")) { |
| info("Posting files to [base] url " + solrUrl + (!auto?" using content-type "+(type==null?DEFAULT_CONTENT_TYPE:type):"")+"..."); |
| if(auto) |
| info("Entering auto mode. File endings considered are "+fileTypes); |
| if(recursive > 0) |
| info("Entering recursive mode, max depth="+recursive+", delay="+delay+"s"); |
| int numFilesPosted = postFiles(args, 0, out, type); |
| info(numFilesPosted + " files indexed."); |
| } |
| } |
| |
| private void doArgsMode() { |
| info("POSTing args to " + solrUrl + "..."); |
| for (String a : args) { |
| postData(stringToStream(a), null, out, type, solrUrl); |
| } |
| } |
| |
| private int doWebMode() { |
| reset(); |
| int numPagesPosted = 0; |
| try { |
| if(type != null) { |
| fatal("Specifying content-type with \"-Ddata=web\" is not supported"); |
| } |
| if (args[0].equals("-")) { |
| // Skip posting url if special param "-" given |
| return 0; |
| } |
| // Set Extracting handler as default |
| solrUrl = appendUrlPath(solrUrl, "/extract"); |
| |
| info("Posting web pages to Solr url "+solrUrl); |
| auto=true; |
| info("Entering auto mode. Indexing pages with content-types corresponding to file endings "+fileTypes); |
| if(recursive > 0) { |
| if(recursive > MAX_WEB_DEPTH) { |
| recursive = MAX_WEB_DEPTH; |
| warn("Too large recursion depth for web mode, limiting to "+MAX_WEB_DEPTH+"..."); |
| } |
| if(delay < DEFAULT_WEB_DELAY) |
| warn("Never crawl an external web site faster than every 10 seconds, your IP will probably be blocked"); |
| info("Entering recursive mode, depth="+recursive+", delay="+delay+"s"); |
| } |
| numPagesPosted = postWebPages(args, 0, out); |
| info(numPagesPosted + " web pages indexed."); |
| } catch(MalformedURLException e) { |
| fatal("Wrong URL trying to append /extract to "+solrUrl); |
| } |
| return numPagesPosted; |
| } |
| |
| private void doStdinMode() { |
| info("POSTing stdin to " + solrUrl + "..."); |
| postData(System.in, null, out, type, solrUrl); |
| } |
| |
| private void reset() { |
| backlog = new ArrayList<>(); |
| visited = new HashSet<>(); |
| } |
| |
| |
| // |
| // USAGE |
| // |
| private static void usageShort() { |
| System.out.println(USAGE_STRING_SHORT+"\n"+ |
| " Please invoke with -h option for extended usage help."); |
| } |
| |
| private static void usage() { |
| System.out.println |
| (USAGE_STRING_SHORT+"\n\n" + |
| "Supported System Properties and their defaults:\n"+ |
| " -Dc=<core/collection>\n"+ |
| " -Durl=<base Solr update URL> (overrides -Dc option if specified)\n"+ |
| " -Ddata=files|web|args|stdin (default=" + DEFAULT_DATA_MODE + ")\n"+ |
| " -Dtype=<content-type> (default=" + DEFAULT_CONTENT_TYPE + ")\n"+ |
| " -Dhost=<host> (default: " + DEFAULT_POST_HOST+ ")\n"+ |
| " -Dport=<port> (default: " + DEFAULT_POST_PORT+ ")\n"+ |
| " -Dbasicauth=<user:pass> (sets Basic Authentication credentials)\n"+ |
| " -Dauto=yes|no (default=" + DEFAULT_AUTO + ")\n"+ |
| " -Drecursive=yes|no|<depth> (default=" + DEFAULT_RECURSIVE + ")\n"+ |
| " -Ddelay=<seconds> (default=0 for files, 10 for web)\n"+ |
| " -Dfiletypes=<type>[,<type>,...] (default=" + DEFAULT_FILE_TYPES + ")\n"+ |
| " -Dparams=\"<key>=<value>[&<key>=<value>...]\" (values must be URL-encoded)\n"+ |
| " -Dcommit=yes|no (default=" + DEFAULT_COMMIT + ")\n"+ |
| " -Doptimize=yes|no (default=" + DEFAULT_OPTIMIZE + ")\n"+ |
| " -Dout=yes|no (default=" + DEFAULT_OUT + ")\n\n"+ |
| "This is a simple command line tool for POSTing raw data to a Solr port.\n"+ |
| "NOTE: Specifying the url/core/collection name is mandatory.\n" + |
| "Data can be read from files specified as commandline args,\n"+ |
| "URLs specified as args, as raw commandline arg strings or via STDIN.\n"+ |
| "Examples:\n"+ |
| " java -Dc=gettingstarted -jar post.jar *.xml\n"+ |
| " java -Ddata=args -Dc=gettingstarted -jar post.jar '<delete><id>42</id></delete>'\n"+ |
| " java -Ddata=stdin -Dc=gettingstarted -jar post.jar < hd.xml\n"+ |
| " java -Ddata=web -Dc=gettingstarted -jar post.jar http://example.com/\n"+ |
| " java -Dtype=text/csv -Dc=gettingstarted -jar post.jar *.csv\n"+ |
| " java -Dtype=application/json -Dc=gettingstarted -jar post.jar *.json\n"+ |
| " java -Durl=http://localhost:8983/solr/techproducts/update/extract -Dparams=literal.id=pdf1 -jar post.jar solr-word.pdf\n"+ |
| " java -Dauto -Dc=gettingstarted -jar post.jar *\n"+ |
| " java -Dauto -Dc=gettingstarted -Drecursive -jar post.jar afolder\n"+ |
| " java -Dauto -Dc=gettingstarted -Dfiletypes=ppt,html -jar post.jar afolder\n"+ |
| "The options controlled by System Properties include the Solr\n"+ |
| "URL to POST to, the Content-Type of the data, whether a commit\n"+ |
| "or optimize should be executed, and whether the response should\n"+ |
| "be written to STDOUT. If auto=yes the tool will try to set type\n"+ |
| "automatically from file name. When posting rich documents the\n"+ |
| "file name will be propagated as \"resource.name\" and also used\n"+ |
| "as \"literal.id\". You may override these or any other request parameter\n"+ |
| "through the -Dparams property. To do a commit only, use \"-\" as argument.\n"+ |
| "The web mode is a simple crawler following links within domain, default delay=10s."); |
| } |
| |
| private boolean checkIsValidPath(File srcFile) { |
| try { |
| srcFile.toPath(); |
| return true; |
| } catch (InvalidPathException e) { |
| return false; |
| } |
| } |
| |
| /** Post all filenames provided in args |
| * @param args array of file names |
| * @param startIndexInArgs offset to start |
| * @param out output stream to post data to |
| * @param type default content-type to use when posting (may be overridden in auto mode) |
| * @return number of files posted |
| * */ |
| public int postFiles(String [] args,int startIndexInArgs, OutputStream out, String type) { |
| reset(); |
| int filesPosted = 0; |
| for (int j = startIndexInArgs; j < args.length; j++) { |
| File srcFile = new File(args[j]); |
| boolean isValidPath = checkIsValidPath(srcFile); |
| if(isValidPath && srcFile.isDirectory() && srcFile.canRead()) { |
| filesPosted += postDirectory(srcFile, out, type); |
| } else if (isValidPath && srcFile.isFile() && srcFile.canRead()) { |
| filesPosted += postFiles(new File[] {srcFile}, out, type); |
| } else { |
| filesPosted += handleGlob(srcFile, out, type); |
| } |
| } |
| return filesPosted; |
| } |
| |
| /** Post all filenames provided in args |
| * @param files array of Files |
| * @param startIndexInArgs offset to start |
| * @param out output stream to post data to |
| * @param type default content-type to use when posting (may be overridden in auto mode) |
| * @return number of files posted |
| * */ |
| public int postFiles(File[] files, int startIndexInArgs, OutputStream out, String type) { |
| reset(); |
| int filesPosted = 0; |
| for (File srcFile : files) { |
| boolean isValidPath = checkIsValidPath(srcFile); |
| if(isValidPath && srcFile.isDirectory() && srcFile.canRead()) { |
| filesPosted += postDirectory(srcFile, out, type); |
| } else if (isValidPath && srcFile.isFile() && srcFile.canRead()) { |
| filesPosted += postFiles(new File[] {srcFile}, out, type); |
| } else { |
| filesPosted += handleGlob(srcFile, out, type); |
| } |
| } |
| return filesPosted; |
| } |
| |
| /** |
| * Posts a whole directory |
| * @return number of files posted total |
| */ |
| private int postDirectory(File dir, OutputStream out, String type) { |
| if(dir.isHidden() && !dir.getName().equals(".")) |
| return(0); |
| info("Indexing directory "+dir.getPath()+" ("+dir.listFiles(fileFilter).length+" files, depth="+currentDepth+")"); |
| int posted = 0; |
| posted += postFiles(dir.listFiles(fileFilter), out, type); |
| if(recursive > currentDepth) { |
| for(File d : dir.listFiles()) { |
| if(d.isDirectory()) { |
| currentDepth++; |
| posted += postDirectory(d, out, type); |
| currentDepth--; |
| } |
| } |
| } |
| return posted; |
| } |
| |
| /** |
| * Posts a list of file names |
| * @return number of files posted |
| */ |
| int postFiles(File[] files, OutputStream out, String type) { |
| int filesPosted = 0; |
| for(File srcFile : files) { |
| try { |
| if(!srcFile.isFile() || srcFile.isHidden()) |
| continue; |
| postFile(srcFile, out, type); |
| Thread.sleep(delay * 1000); |
| filesPosted++; |
| } catch (InterruptedException e) { |
| throw new RuntimeException(e); |
| } |
| } |
| return filesPosted; |
| } |
| |
| /** |
| * This only handles file globs not full path globbing. |
| * @param globFile file holding glob path |
| * @param out outputStream to write results to |
| * @param type default content-type to use when posting (may be overridden in auto mode) |
| * @return number of files posted |
| */ |
| int handleGlob(File globFile, OutputStream out, String type) { |
| int filesPosted = 0; |
| File parent = globFile.getParentFile(); |
| if (parent == null) parent = new File("."); |
| String fileGlob = globFile.getName(); |
| GlobFileFilter ff = new GlobFileFilter(fileGlob, false); |
| File[] fileList = parent.listFiles(ff); |
| if (fileList == null || fileList.length == 0) { |
| warn("No files or directories matching " + globFile); |
| } else { |
| filesPosted = postFiles(fileList, out, type); |
| } |
| return filesPosted; |
| } |
| |
| /** |
| * This method takes as input a list of start URL strings for crawling, |
| * adds each one to a backlog and then starts crawling |
| * @param args the raw input args from main() |
| * @param startIndexInArgs offset for where to start |
| * @param out outputStream to write results to |
| * @return the number of web pages posted |
| */ |
| public int postWebPages(String[] args, int startIndexInArgs, OutputStream out) { |
| reset(); |
| LinkedHashSet<URL> s = new LinkedHashSet<>(); |
| for (int j = startIndexInArgs; j < args.length; j++) { |
| try { |
| URL u = new URL(normalizeUrlEnding(args[j])); |
| s.add(u); |
| } catch(MalformedURLException e) { |
| warn("Skipping malformed input URL: "+args[j]); |
| } |
| } |
| // Add URLs to level 0 of the backlog and start recursive crawling |
| backlog.add(s); |
| return webCrawl(0, out); |
| } |
| |
| /** |
| * Normalizes a URL string by removing anchor part and trailing slash |
| * @return the normalized URL string |
| */ |
| protected static String normalizeUrlEnding(String link) { |
| if(link.indexOf("#") > -1) |
| link = link.substring(0,link.indexOf("#")); |
| if(link.endsWith("?")) |
| link = link.substring(0,link.length()-1); |
| if(link.endsWith("/")) |
| link = link.substring(0,link.length()-1); |
| return link; |
| } |
| |
| /** |
| * A very simple crawler, pulling URLs to fetch from a backlog and then |
| * recurses N levels deep if recursive>0. Links are parsed from HTML |
| * through first getting an XHTML version using SolrCell with extractOnly, |
| * and followed if they are local. The crawler pauses for a default delay |
| * of 10 seconds between each fetch, this can be configured in the delay |
| * variable. This is only meant for test purposes, as it does not respect |
| * robots or anything else fancy :) |
| * @param level which level to crawl |
| * @param out output stream to write to |
| * @return number of pages crawled on this level and below |
| */ |
| protected int webCrawl(int level, OutputStream out) { |
| int numPages = 0; |
| LinkedHashSet<URL> stack = backlog.get(level); |
| int rawStackSize = stack.size(); |
| stack.removeAll(visited); |
| int stackSize = stack.size(); |
| LinkedHashSet<URL> subStack = new LinkedHashSet<>(); |
| info("Entering crawl at level "+level+" ("+rawStackSize+" links total, "+stackSize+" new)"); |
| for(URL u : stack) { |
| try { |
| visited.add(u); |
| PageFetcherResult result = pageFetcher.readPageFromUrl(u); |
| if(result.httpStatus == 200) { |
| u = (result.redirectUrl != null) ? result.redirectUrl : u; |
| URL postUrl = new URL(appendParam(solrUrl.toString(), |
| "literal.id="+URLEncoder.encode(u.toString(),"UTF-8") + |
| "&literal.url="+URLEncoder.encode(u.toString(),"UTF-8"))); |
| ByteBuffer content = result.content; |
| boolean success = postData(new ByteArrayInputStream(content.array(), content.arrayOffset(), content.limit()), null, out, result.contentType, postUrl); |
| if (success) { |
| info("POSTed web resource "+u+" (depth: "+level+")"); |
| Thread.sleep(delay * 1000); |
| numPages++; |
| // Pull links from HTML pages only |
| if(recursive > level && result.contentType.equals("text/html")) { |
| Set<URL> children = pageFetcher.getLinksFromWebPage(u, new ByteArrayInputStream(content.array(), content.arrayOffset(), content.limit()), result.contentType, postUrl); |
| subStack.addAll(children); |
| } |
| } else { |
| warn("An error occurred while posting "+u); |
| } |
| } else { |
| warn("The URL "+u+" returned a HTTP result status of "+result.httpStatus); |
| } |
| } catch (IOException e) { |
| warn("Caught exception when trying to open connection to "+u+": "+e.getMessage()); |
| } catch (InterruptedException e) { |
| throw new RuntimeException(e); |
| } |
| } |
| if(!subStack.isEmpty()) { |
| backlog.add(subStack); |
| numPages += webCrawl(level+1, out); |
| } |
| return numPages; |
| } |
| public static class BAOS extends ByteArrayOutputStream { |
| public ByteBuffer getByteBuffer() { |
| return ByteBuffer.wrap(super.buf,0,super.count); |
| } |
| } |
| public static ByteBuffer inputStreamToByteArray(InputStream is) throws IOException { |
| return inputStreamToByteArray(is,Integer.MAX_VALUE); |
| |
| } |
| |
| /** |
| * Reads an input stream into a byte array |
| * |
| * @param is the input stream |
| * @return the byte array |
| * @throws IOException If there is a low-level I/O error. |
| */ |
| public static ByteBuffer inputStreamToByteArray(InputStream is, long maxSize) throws IOException { |
| try (BAOS bos = new BAOS()) { |
| long sz = 0; |
| int next = is.read(); |
| while (next > -1) { |
| if (++sz > maxSize) throw new BufferOverflowException(); |
| bos.write(next); |
| next = is.read(); |
| } |
| bos.flush(); |
| return bos.getByteBuffer(); |
| } |
| } |
| |
| /** |
| * Computes the full URL based on a base url and a possibly relative link found |
| * in the href param of an HTML anchor. |
| * @param baseUrl the base url from where the link was found |
| * @param link the absolute or relative link |
| * @return the string version of the full URL |
| */ |
| protected String computeFullUrl(URL baseUrl, String link) { |
| if(link == null || link.length() == 0) { |
| return null; |
| } |
| if(!link.startsWith("http")) { |
| if(link.startsWith("/")) { |
| link = baseUrl.getProtocol() + "://" + baseUrl.getAuthority() + link; |
| } else { |
| if(link.contains(":")) { |
| return null; // Skip non-relative URLs |
| } |
| String path = baseUrl.getPath(); |
| if(!path.endsWith("/")) { |
| int sep = path.lastIndexOf("/"); |
| String file = path.substring(sep+1); |
| if(file.contains(".") || file.contains("?")) |
| path = path.substring(0,sep); |
| } |
| link = baseUrl.getProtocol() + "://" + baseUrl.getAuthority() + path + "/" + link; |
| } |
| } |
| link = normalizeUrlEnding(link); |
| String l = link.toLowerCase(Locale.ROOT); |
| // Simple brute force skip images |
| if(l.endsWith(".jpg") || l.endsWith(".jpeg") || l.endsWith(".png") || l.endsWith(".gif")) { |
| return null; // Skip images |
| } |
| return link; |
| } |
| |
| /** |
| * Uses the mime-type map to reverse lookup whether the file ending for our type |
| * is supported by the fileTypes option |
| * @param type what content-type to lookup |
| * @return true if this is a supported content type |
| */ |
| protected boolean typeSupported(String type) { |
| for(Map.Entry<String, String> entry : mimeMap.entrySet()) { |
| if(entry.getValue().equals(type)) { |
| if(fileTypes.contains(entry.getKey())) |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| /** |
| * Tests if a string is either "true", "on", "yes" or "1" |
| * @param property the string to test |
| * @return true if "on" |
| */ |
| protected static boolean isOn(String property) { |
| return("true,on,yes,1".indexOf(property) > -1); |
| } |
| |
| static void warn(String msg) { |
| System.err.println("SimplePostTool: WARNING: " + msg); |
| } |
| |
| static void info(String msg) { |
| System.out.println(msg); |
| } |
| |
| static void fatal(String msg) { |
| System.err.println("SimplePostTool: FATAL: " + msg); |
| System.exit(2); |
| } |
| |
| /** |
| * Does a simple commit operation |
| */ |
| public void commit() { |
| info("COMMITting Solr index changes to " + solrUrl + "..."); |
| doGet(appendParam(solrUrl.toString(), "commit=true")); |
| } |
| |
| /** |
| * Does a simple optimize operation |
| */ |
| public void optimize() { |
| info("Performing an OPTIMIZE to " + solrUrl + "..."); |
| doGet(appendParam(solrUrl.toString(), "optimize=true")); |
| } |
| |
| /** |
| * Appends a URL query parameter to a URL |
| * @param url the original URL |
| * @param param the parameter(s) to append, separated by "&" |
| * @return the string version of the resulting URL |
| */ |
| public static String appendParam(String url, String param) { |
| String[] pa = param.split("&"); |
| for(String p : pa) { |
| if(p.trim().length() == 0) continue; |
| String[] kv = p.split("="); |
| if(kv.length == 2) { |
| url = url + (url.indexOf('?')>0 ? "&" : "?") + kv[0] +"="+ kv[1]; |
| } else { |
| warn("Skipping param "+p+" which is not on form key=value"); |
| } |
| } |
| return url; |
| } |
| |
| /** |
| * Opens the file and posts its contents to the solrUrl, |
| * writes to response to output. |
| */ |
| public void postFile(File file, OutputStream output, String type) { |
| InputStream is = null; |
| try { |
| URL url = solrUrl; |
| String suffix = ""; |
| if(auto) { |
| if(type == null) { |
| type = guessType(file); |
| } |
| // TODO: Add a flag that disables /update and sends all to /update/extract, to avoid CSV, JSON, and XML files |
| // TODO: from being interpreted as Solr documents internally |
| if (type.equals("application/json") && !"solr".equals(format)) { |
| suffix = "/json/docs"; |
| String urlStr = appendUrlPath(solrUrl, suffix).toString(); |
| url = new URL(urlStr); |
| } else if (type.equals("application/xml") || type.equals("text/csv") || type.equals("application/json")) { |
| // Default handler |
| } else { |
| // SolrCell |
| suffix = "/extract"; |
| String urlStr = appendUrlPath(solrUrl, suffix).toString(); |
| if(urlStr.indexOf("resource.name")==-1) |
| urlStr = appendParam(urlStr, "resource.name=" + URLEncoder.encode(file.getAbsolutePath(), "UTF-8")); |
| if(urlStr.indexOf("literal.id")==-1) |
| urlStr = appendParam(urlStr, "literal.id=" + URLEncoder.encode(file.getAbsolutePath(), "UTF-8")); |
| url = new URL(urlStr); |
| } |
| } else { |
| if(type == null) type = DEFAULT_CONTENT_TYPE; |
| } |
| info("POSTing file " + file.getName() + (auto?" ("+type+")":"") + " to [base]" + suffix + (mockMode ? " MOCK!":"")); |
| is = new FileInputStream(file); |
| postData(is, file.length(), output, type, url); |
| } catch (IOException e) { |
| e.printStackTrace(); |
| warn("Can't open/read file: " + file); |
| } finally { |
| try { |
| if(is!=null) is.close(); |
| } catch (IOException e) { |
| fatal("IOException while closing file: "+ e); |
| } |
| } |
| } |
| |
| /** |
| * Appends to the path of the URL |
| * @param url the URL |
| * @param append the path to append |
| * @return the final URL version |
| */ |
| protected static URL appendUrlPath(URL url, String append) throws MalformedURLException { |
| return new URL(url.getProtocol() + "://" + url.getAuthority() + url.getPath() + append + (url.getQuery() != null ? "?"+url.getQuery() : "")); |
| } |
| |
| /** |
| * Guesses the type of a file, based on file name suffix |
| * Returns "application/octet-stream" if no corresponding mimeMap type. |
| * @param file the file |
| * @return the content-type guessed |
| */ |
| protected static String guessType(File file) { |
| String name = file.getName(); |
| String suffix = name.substring(name.lastIndexOf(".")+1); |
| String type = mimeMap.get(suffix.toLowerCase(Locale.ROOT)); |
| return (type != null) ? type : "application/octet-stream"; |
| } |
| |
| /** |
| * Performs a simple get on the given URL |
| */ |
| public void doGet(String url) { |
| try { |
| doGet(new URL(url)); |
| } catch (MalformedURLException e) { |
| warn("The specified URL "+url+" is not a valid URL. Please check"); |
| } |
| } |
| |
| /** |
| * Performs a simple get on the given URL |
| */ |
| public void doGet(URL url) { |
| try { |
| if(mockMode) return; |
| HttpURLConnection urlc = (HttpURLConnection) url.openConnection(); |
| basicAuth(urlc); |
| urlc.connect(); |
| checkResponseCode(urlc); |
| } catch (IOException e) { |
| warn("An error occurred getting data from "+url+". Please check that Solr is running."); |
| } catch (Exception e) { |
| warn("An error occurred getting data from "+url+". Message: " + e.getMessage()); |
| } |
| } |
| |
| /** |
| * Reads data from the data stream and posts it to solr, |
| * writes to the response to output |
| * @return true if success |
| */ |
| public boolean postData(InputStream data, Long length, OutputStream output, String type, URL url) { |
| if(mockMode) return true; |
| boolean success = true; |
| if(type == null) |
| type = DEFAULT_CONTENT_TYPE; |
| HttpURLConnection urlc = null; |
| try { |
| try { |
| urlc = (HttpURLConnection) url.openConnection(); |
| try { |
| urlc.setRequestMethod("POST"); |
| } catch (ProtocolException e) { |
| fatal("Shouldn't happen: HttpURLConnection doesn't support POST??"+e); |
| } |
| urlc.setDoOutput(true); |
| urlc.setDoInput(true); |
| urlc.setUseCaches(false); |
| urlc.setAllowUserInteraction(false); |
| urlc.setRequestProperty("Content-type", type); |
| basicAuth(urlc); |
| if (null != length) { |
| urlc.setFixedLengthStreamingMode(length); |
| } else { |
| urlc.setChunkedStreamingMode(-1);//use JDK default chunkLen, 4k in Java 8. |
| } |
| urlc.connect(); |
| } catch (IOException e) { |
| fatal("Connection error (is Solr running at " + solrUrl + " ?): " + e); |
| success = false; |
| } catch (Exception e) { |
| fatal("POST failed with error " + e.getMessage()); |
| } |
| |
| try (final OutputStream out = urlc.getOutputStream()) { |
| pipe(data, out); |
| } catch (IOException e) { |
| fatal("IOException while posting data: " + e); |
| } |
| |
| try { |
| success &= checkResponseCode(urlc); |
| try (final InputStream in = urlc.getInputStream()) { |
| pipe(in, output); |
| } |
| } catch (IOException e) { |
| warn("IOException while reading response: " + e); |
| success = false; |
| } catch (GeneralSecurityException e) { |
| fatal("Looks like Solr is secured and would not let us in. Try with another user in '-u' parameter"); |
| } |
| } finally { |
| if (urlc!=null) urlc.disconnect(); |
| } |
| return success; |
| } |
| |
| private static void basicAuth(HttpURLConnection urlc) throws Exception { |
| if (urlc.getURL().getUserInfo() != null) { |
| String encoding = Base64.getEncoder().encodeToString(urlc.getURL().getUserInfo().getBytes(US_ASCII)); |
| urlc.setRequestProperty("Authorization", "Basic " + encoding); |
| } else if (System.getProperty(BASIC_AUTH) != null) { |
| String basicauth = System.getProperty(BASIC_AUTH).trim(); |
| if (!basicauth.contains(":")) { |
| throw new Exception("System property '"+BASIC_AUTH+"' must be of format user:pass"); |
| } |
| urlc.setRequestProperty("Authorization", "Basic " + Base64.getEncoder().encodeToString(basicauth.getBytes(UTF_8))); |
| } |
| } |
| |
| private static boolean checkResponseCode(HttpURLConnection urlc) throws IOException, GeneralSecurityException { |
| if (urlc.getResponseCode() >= 400) { |
| warn("Solr returned an error #" + urlc.getResponseCode() + |
| " (" + urlc.getResponseMessage() + ") for url: " + urlc.getURL()); |
| Charset charset = StandardCharsets.ISO_8859_1; |
| final String contentType = urlc.getContentType(); |
| // code cloned from ContentStreamBase, but post.jar should be standalone! |
| if (contentType != null) { |
| int idx = contentType.toLowerCase(Locale.ROOT).indexOf("charset="); |
| if (idx > 0) { |
| charset = Charset.forName(contentType.substring(idx + "charset=".length()).trim()); |
| } |
| } |
| // Print the response returned by Solr |
| try (InputStream errStream = urlc.getErrorStream()) { |
| if (errStream != null) { |
| BufferedReader br = new BufferedReader(new InputStreamReader(errStream, charset)); |
| final StringBuilder response = new StringBuilder("Response: "); |
| int ch; |
| while ((ch = br.read()) != -1) { |
| response.append((char) ch); |
| } |
| warn(response.toString().trim()); |
| } |
| } |
| if (urlc.getResponseCode() == 401) { |
| throw new GeneralSecurityException("Solr requires authentication (response 401). Please try again with '-u' option"); |
| } |
| if (urlc.getResponseCode() == 403) { |
| throw new GeneralSecurityException("You are not authorized to perform this action against Solr. (response 403)"); |
| } |
| return false; |
| } |
| return true; |
| } |
| |
| /** |
| * Converts a string to an input stream |
| * @param s the string |
| * @return the input stream |
| */ |
| public static InputStream stringToStream(String s) { |
| return new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8)); |
| } |
| |
| /** |
| * Pipes everything from the source to the dest. If dest is null, |
| * then everything is read from source and thrown away. |
| */ |
| private static void pipe(InputStream source, OutputStream dest) throws IOException { |
| byte[] buf = new byte[1024]; |
| int read = 0; |
| while ( (read = source.read(buf) ) >= 0) { |
| if (null != dest) dest.write(buf, 0, read); |
| } |
| if (null != dest) dest.flush(); |
| } |
| |
| public FileFilter getFileFilterFromFileTypes(String fileTypes) { |
| String glob; |
| if(fileTypes.equals("*")) |
| glob = ".*"; |
| else |
| glob = "^.*\\.(" + fileTypes.replace(",", "|") + ")$"; |
| return new GlobFileFilter(glob, true); |
| } |
| |
| // |
| // Utility methods for XPath handing |
| // |
| |
| /** |
| * Gets all nodes matching an XPath |
| */ |
| public static NodeList getNodesFromXP(Node n, String xpath) throws XPathExpressionException { |
| XPathFactory factory = XPathFactory.newInstance(); |
| XPath xp = factory.newXPath(); |
| XPathExpression expr = xp.compile(xpath); |
| return (NodeList) expr.evaluate(n, XPathConstants.NODESET); |
| } |
| |
| /** |
| * Gets the string content of the matching an XPath |
| * @param n the node (or doc) |
| * @param xpath the xpath string |
| * @param concatAll if true, text from all matching nodes will be concatenated, else only the first returned |
| */ |
| public static String getXP(Node n, String xpath, boolean concatAll) |
| throws XPathExpressionException { |
| NodeList nodes = getNodesFromXP(n, xpath); |
| StringBuilder sb = new StringBuilder(); |
| if (nodes.getLength() > 0) { |
| for(int i = 0; i < nodes.getLength() ; i++) { |
| sb.append(nodes.item(i).getNodeValue()).append(' '); |
| if(!concatAll) break; |
| } |
| return sb.toString().trim(); |
| } else |
| return ""; |
| } |
| |
| /** |
| * Takes a string as input and returns a DOM |
| */ |
| public static Document makeDom(byte[] in) throws SAXException, IOException, |
| ParserConfigurationException { |
| InputStream is = new ByteArrayInputStream(in); |
| Document dom = DocumentBuilderFactory.newInstance() |
| .newDocumentBuilder().parse(is); |
| return dom; |
| } |
| |
| /** |
| * Inner class to filter files based on glob wildcards |
| */ |
| static class GlobFileFilter implements FileFilter |
| { |
| private String _pattern; |
| private Pattern p; |
| |
| public GlobFileFilter(String pattern, boolean isRegex) |
| { |
| _pattern = pattern; |
| if(!isRegex) { |
| _pattern = _pattern |
| .replace("^", "\\^") |
| .replace("$", "\\$") |
| .replace(".", "\\.") |
| .replace("(", "\\(") |
| .replace(")", "\\)") |
| .replace("+", "\\+") |
| .replace("*", ".*") |
| .replace("?", "."); |
| _pattern = "^" + _pattern + "$"; |
| } |
| |
| try { |
| p = Pattern.compile(_pattern,Pattern.CASE_INSENSITIVE); |
| } catch(PatternSyntaxException e) { |
| fatal("Invalid type list "+pattern+". "+e.getDescription()); |
| } |
| } |
| |
| @Override |
| public boolean accept(File file) |
| { |
| return p.matcher(file.getName()).find(); |
| } |
| } |
| |
| // |
| // Simple crawler class which can fetch a page and check for robots.txt |
| // |
| class PageFetcher { |
| Map<String, List<String>> robotsCache; |
| static final String DISALLOW = "Disallow:"; |
| |
| public PageFetcher() { |
| robotsCache = new HashMap<>(); |
| } |
| |
| public PageFetcherResult readPageFromUrl(URL u) { |
| PageFetcherResult res = new PageFetcherResult(); |
| try { |
| if (isDisallowedByRobots(u)) { |
| warn("The URL "+u+" is disallowed by robots.txt and will not be crawled."); |
| res.httpStatus = 403; |
| visited.add(u); |
| return res; |
| } |
| res.httpStatus = 404; |
| HttpURLConnection conn = (HttpURLConnection) u.openConnection(); |
| conn.setRequestProperty("User-Agent", "SimplePostTool-crawler/"+VERSION_OF_THIS_TOOL+" (http://lucene.apache.org/solr/)"); |
| conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); |
| conn.connect(); |
| res.httpStatus = conn.getResponseCode(); |
| if(!normalizeUrlEnding(conn.getURL().toString()).equals(normalizeUrlEnding(u.toString()))) { |
| info("The URL "+u+" caused a redirect to "+conn.getURL()); |
| u = conn.getURL(); |
| res.redirectUrl = u; |
| visited.add(u); |
| } |
| if(res.httpStatus == 200) { |
| // Raw content type of form "text/html; encoding=utf-8" |
| String rawContentType = conn.getContentType(); |
| String type = rawContentType.split(";")[0]; |
| if(typeSupported(type) || "*".equals(fileTypes)) { |
| String encoding = conn.getContentEncoding(); |
| InputStream is; |
| if (encoding != null && encoding.equalsIgnoreCase("gzip")) { |
| is = new GZIPInputStream(conn.getInputStream()); |
| } else if (encoding != null && encoding.equalsIgnoreCase("deflate")) { |
| is = new InflaterInputStream(conn.getInputStream(), new Inflater(true)); |
| } else { |
| is = conn.getInputStream(); |
| } |
| |
| // Read into memory, so that we later can pull links from the page without re-fetching |
| res.content = inputStreamToByteArray(is); |
| is.close(); |
| } else { |
| warn("Skipping URL with unsupported type "+type); |
| res.httpStatus = 415; |
| } |
| } |
| } catch(IOException e) { |
| warn("IOException when reading page from url "+u+": "+e.getMessage()); |
| } |
| return res; |
| } |
| |
| public boolean isDisallowedByRobots(URL url) { |
| String host = url.getHost(); |
| String strRobot = url.getProtocol() + "://" + host + "/robots.txt"; |
| List<String> disallows = robotsCache.get(host); |
| if(disallows == null) { |
| disallows = new ArrayList<>(); |
| URL urlRobot; |
| try { |
| urlRobot = new URL(strRobot); |
| disallows = parseRobotsTxt(urlRobot.openStream()); |
| } catch (MalformedURLException e) { |
| return true; // We cannot trust this robots URL, should not happen |
| } catch (IOException e) { |
| // There is no robots.txt, will cache an empty disallow list |
| } |
| } |
| |
| robotsCache.put(host, disallows); |
| |
| String strURL = url.getFile(); |
| for (String path : disallows) { |
| if (path.equals("/") || strURL.indexOf(path) == 0) |
| return true; |
| } |
| return false; |
| } |
| |
| /** |
| * Very simple robots.txt parser which obeys all Disallow lines regardless |
| * of user agent or whether there are valid Allow: lines. |
| * @param is Input stream of the robots.txt file |
| * @return a list of disallow paths |
| * @throws IOException if problems reading the stream |
| */ |
| protected List<String> parseRobotsTxt(InputStream is) throws IOException { |
| List<String> disallows = new ArrayList<>(); |
| BufferedReader r = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); |
| String l; |
| while((l = r.readLine()) != null) { |
| String[] arr = l.split("#"); |
| if(arr.length == 0) continue; |
| l = arr[0].trim(); |
| if(l.startsWith(DISALLOW)) { |
| l = l.substring(DISALLOW.length()).trim(); |
| if(l.length() == 0) continue; |
| disallows.add(l); |
| } |
| } |
| is.close(); |
| return disallows; |
| } |
| |
| /** |
| * Finds links on a web page, using /extract?extractOnly=true |
| * @param u the URL of the web page |
| * @param is the input stream of the page |
| * @param type the content-type |
| * @param postUrl the URL (typically /solr/extract) in order to pull out links |
| * @return a set of URLs parsed from the page |
| */ |
| protected Set<URL> getLinksFromWebPage(URL u, InputStream is, String type, URL postUrl) { |
| Set<URL> l = new HashSet<>(); |
| URL url = null; |
| try { |
| ByteArrayOutputStream os = new ByteArrayOutputStream(); |
| URL extractUrl = new URL(appendParam(postUrl.toString(), "extractOnly=true")); |
| boolean success = postData(is, null, os, type, extractUrl); |
| if(success) { |
| Document d = makeDom(os.toByteArray()); |
| String innerXml = getXP(d, "/response/str/text()[1]", false); |
| d = makeDom(innerXml.getBytes(StandardCharsets.UTF_8)); |
| NodeList links = getNodesFromXP(d, "/html/body//a/@href"); |
| for(int i = 0; i < links.getLength(); i++) { |
| String link = links.item(i).getTextContent(); |
| link = computeFullUrl(u, link); |
| if(link == null) |
| continue; |
| url = new URL(link); |
| if(url.getAuthority() == null || !url.getAuthority().equals(u.getAuthority())) |
| continue; |
| l.add(url); |
| } |
| } |
| } catch (MalformedURLException e) { |
| warn("Malformed URL "+url); |
| } catch (IOException e) { |
| warn("IOException opening URL "+url+": "+e.getMessage()); |
| } catch (Exception e) { |
| throw new RuntimeException(e); |
| } |
| return l; |
| } |
| } |
| |
| /** |
| * Utility class to hold the result form a page fetch |
| */ |
| public static class PageFetcherResult { |
| int httpStatus = 200; |
| String contentType = "text/html"; |
| URL redirectUrl = null; |
| ByteBuffer content; |
| } |
| } |
