commit 39a622fcbb8705f66a3acc6b9160734307b188fc
author Keith Turner <kturner@apache.org> Thu Apr 06 18:49:02 2017 -0400
committer Keith Turner <kturner@apache.org> Thu Apr 06 18:49:02 2017 -0400
tree e01d1fecc1ecbf53aa01d69943c51d90578f2d89
parent 2e22b469c2b5bc7c13ac81604806e61c818eb529

update Common Crawl S3 URLs

bin/webindex

diff --git a/bin/webindex b/bin/webindex
index 2d56167..eec8ba0 100755
--- a/bin/webindex
+++ b/bin/webindex
@@ -64,7 +64,7 @@
   PATHS_FILE="$2".wat.paths
   if [ ! -f $DATA_DIR/$PATHS_FILE ]; then
     rm -f $DATA_DIR/wat.paths.gz
-    PATHS_URL=https://aws-publicdatasets.s3.amazonaws.com/common-crawl/crawl-data/CC-MAIN-$2/wat.paths.gz
+    PATHS_URL=https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-$2/wat.paths.gz
     if [[ `wget -S --spider $PATHS_URL 2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then
       wget -P $DATA_DIR $PATHS_URL
       gzip -d $DATA_DIR/wat.paths.gz

modules/core/src/main/java/webindex/core/WebIndexConfig.java

diff --git a/modules/core/src/main/java/webindex/core/WebIndexConfig.java b/modules/core/src/main/java/webindex/core/WebIndexConfig.java
index dadc429..0e9840f 100644
--- a/modules/core/src/main/java/webindex/core/WebIndexConfig.java
+++ b/modules/core/src/main/java/webindex/core/WebIndexConfig.java
@@ -25,7 +25,7 @@
 
   private static final Logger log = LoggerFactory.getLogger(WebIndexConfig.class);
 
-  public static String CC_URL_PREFIX = "https://aws-publicdatasets.s3.amazonaws.com/";
+  public static String CC_URL_PREFIX = "https://commoncrawl.s3.amazonaws.com/";
   public static final String WI_EXECUTOR_INSTANCES = "WI_EXECUTOR_INSTANCES";
 
   public String fluoHome;