Renamed package names from io.fluo.webindex.* to webindex.*
diff --git a/bin/impl/init.sh b/bin/impl/init.sh
index 75f7216..d1c463c 100755
--- a/bin/impl/init.sh
+++ b/bin/impl/init.sh
@@ -57,11 +57,11 @@
cp $WI_HOME/modules/core/target/webindex-core-0.0.1-SNAPSHOT.jar $FLUO_APP_LIB
mvn dependency:get -Dartifact=commons-validator:commons-validator:1.4.1:jar -Ddest=$FLUO_APP_LIB
-java -cp $WI_DATA_DEP_JAR io.fluo.webindex.data.Configure $DATA_CONFIG
+java -cp $WI_DATA_DEP_JAR webindex.data.Configure $DATA_CONFIG
$FLUO_CMD init $FLUO_APP --force
-$SPARK_SUBMIT --class io.fluo.webindex.data.Init $COMMON_SPARK_OPTS \
+$SPARK_SUBMIT --class webindex.data.Init $COMMON_SPARK_OPTS \
--conf spark.shuffle.service.enabled=true \
--conf spark.executor.extraJavaOptions=-XX:+UseCompressedOops \
$WI_DATA_DEP_JAR $1
diff --git a/bin/webindex b/bin/webindex
index ab6fc9e..5bc6748 100755
--- a/bin/webindex
+++ b/bin/webindex
@@ -82,7 +82,7 @@
exit 1
fi
. $BIN_DIR/impl/base.sh
- COMMAND="$SPARK_SUBMIT --class io.fluo.webindex.data.Copy $COMMON_SPARK_OPTS \
+ COMMAND="$SPARK_SUBMIT --class webindex.data.Copy $COMMON_SPARK_OPTS \
$WI_DATA_DEP_JAR $WI_HOME/paths/"$2".wat.paths $3 $4"
if [ "$5" != "-fg" ]; then
nohup ${COMMAND} &> $COMMAND_LOGFILE &
@@ -119,7 +119,7 @@
echo "Fluo properties file must exist at $FLUO_PROPS"
exit 1
fi
- COMMAND="$SPARK_SUBMIT --class io.fluo.webindex.data.LoadHdfs $COMMON_SPARK_OPTS \
+ COMMAND="$SPARK_SUBMIT --class webindex.data.LoadHdfs $COMMON_SPARK_OPTS \
--files $FLUO_PROPS $WI_DATA_DEP_JAR $2"
if [ "$3" != "-fg" ]; then
nohup ${COMMAND} &> $COMMAND_LOGFILE &
@@ -139,7 +139,7 @@
echo "Fluo properties file must exist at $FLUO_PROPS"
exit 1
fi
- COMMAND="$SPARK_SUBMIT --class io.fluo.webindex.data.LoadS3 $COMMON_SPARK_OPTS \
+ COMMAND="$SPARK_SUBMIT --class webindex.data.LoadS3 $COMMON_SPARK_OPTS \
--files $FLUO_PROPS $WI_DATA_DEP_JAR $WI_HOME/paths/"$2".wat.paths $3"
if [ "$4" != "-fg" ]; then
nohup ${COMMAND} &> $COMMAND_LOGFILE &
@@ -154,7 +154,7 @@
exit 1
fi
. $BIN_DIR/impl/base.sh
- COMMAND="$SPARK_SUBMIT --class io.fluo.webindex.data.TestParser $COMMON_SPARK_OPTS \
+ COMMAND="$SPARK_SUBMIT --class webindex.data.TestParser $COMMON_SPARK_OPTS \
$WI_DATA_DEP_JAR $WI_HOME/paths/"$2".wat.paths $3"
if [ "$4" != "-fg" ]; then
nohup ${COMMAND} &> $COMMAND_LOGFILE &
@@ -185,7 +185,7 @@
;;
splits)
. $BIN_DIR/impl/base.sh
- COMMAND="$SPARK_SUBMIT --class io.fluo.webindex.data.CalcSplits \
+ COMMAND="$SPARK_SUBMIT --class webindex.data.CalcSplits \
$COMMON_SPARK_OPTS \
--conf spark.shuffle.service.enabled=true \
$WI_DATA_DEP_JAR $2"
diff --git a/docs/code-guide.md b/docs/code-guide.md
index b621b8a..5dab8ab 100644
--- a/docs/code-guide.md
+++ b/docs/code-guide.md
@@ -91,13 +91,13 @@
**Code:** [DomainExport.java][DomainExport]
-[PageLoader]: ../modules/data/src/main/java/io/fluo/webindex/data/fluo/PageLoader.java
-[PageObserver]: ../modules/data/src/main/java/io/fluo/webindex/data/fluo/PageObserver.java
-[UriMap]: ../modules/data/src/main/java/io/fluo/webindex/data/fluo/UriMap.java
-[DomainMap]: ../modules/data/src/main/java/io/fluo/webindex/data/fluo/DomainMap.java
-[UriCountExport]: ../modules/data/src/main/java/io/fluo/webindex/data/fluo/UriCountExport.java
-[PageExport]: ../modules/data/src/main/java/io/fluo/webindex/data/fluo/PageExport.java
-[DomainExport]: ../modules/data/src/main/java/io/fluo/webindex/data/fluo/DomainExport.java
+[PageLoader]: ../modules/data/src/main/java/webindex/data/fluo/PageLoader.java
+[PageObserver]: ../modules/data/src/main/java/webindex/data/fluo/PageObserver.java
+[UriMap]: ../modules/data/src/main/java/webindex/data/fluo/UriMap.java
+[DomainMap]: ../modules/data/src/main/java/webindex/data/fluo/DomainMap.java
+[UriCountExport]: ../modules/data/src/main/java/webindex/data/fluo/UriCountExport.java
+[PageExport]: ../modules/data/src/main/java/webindex/data/fluo/PageExport.java
+[DomainExport]: ../modules/data/src/main/java/webindex/data/fluo/DomainExport.java
[qt]: tables.md#query-table-schema
[tables]: tables.md
diff --git a/modules/core/src/main/java/io/fluo/webindex/core/Constants.java b/modules/core/src/main/java/webindex/core/Constants.java
similarity index 97%
rename from modules/core/src/main/java/io/fluo/webindex/core/Constants.java
rename to modules/core/src/main/java/webindex/core/Constants.java
index 7c35e1d..c41eee3 100644
--- a/modules/core/src/main/java/io/fluo/webindex/core/Constants.java
+++ b/modules/core/src/main/java/webindex/core/Constants.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core;
+package webindex.core;
public class Constants {
diff --git a/modules/core/src/main/java/io/fluo/webindex/core/DataConfig.java b/modules/core/src/main/java/webindex/core/DataConfig.java
similarity index 98%
rename from modules/core/src/main/java/io/fluo/webindex/core/DataConfig.java
rename to modules/core/src/main/java/webindex/core/DataConfig.java
index 444b949..36a3bf9 100644
--- a/modules/core/src/main/java/io/fluo/webindex/core/DataConfig.java
+++ b/modules/core/src/main/java/webindex/core/DataConfig.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core;
+package webindex.core;
import java.io.File;
import java.io.FileReader;
diff --git a/modules/core/src/main/java/io/fluo/webindex/core/models/DomainStats.java b/modules/core/src/main/java/webindex/core/models/DomainStats.java
similarity index 95%
rename from modules/core/src/main/java/io/fluo/webindex/core/models/DomainStats.java
rename to modules/core/src/main/java/webindex/core/models/DomainStats.java
index 06b2048..7459f4f 100644
--- a/modules/core/src/main/java/io/fluo/webindex/core/models/DomainStats.java
+++ b/modules/core/src/main/java/webindex/core/models/DomainStats.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core.models;
+package webindex.core.models;
public class DomainStats {
diff --git a/modules/core/src/main/java/io/fluo/webindex/core/models/Link.java b/modules/core/src/main/java/webindex/core/models/Link.java
similarity index 97%
rename from modules/core/src/main/java/io/fluo/webindex/core/models/Link.java
rename to modules/core/src/main/java/webindex/core/models/Link.java
index 72439cb..6544224 100644
--- a/modules/core/src/main/java/io/fluo/webindex/core/models/Link.java
+++ b/modules/core/src/main/java/webindex/core/models/Link.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core.models;
+package webindex.core.models;
import java.io.Serializable;
import java.util.Objects;
diff --git a/modules/core/src/main/java/io/fluo/webindex/core/models/Links.java b/modules/core/src/main/java/webindex/core/models/Links.java
similarity index 97%
rename from modules/core/src/main/java/io/fluo/webindex/core/models/Links.java
rename to modules/core/src/main/java/webindex/core/models/Links.java
index 12d8cd3..c870852 100644
--- a/modules/core/src/main/java/io/fluo/webindex/core/models/Links.java
+++ b/modules/core/src/main/java/webindex/core/models/Links.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core.models;
+package webindex.core.models;
import java.util.ArrayList;
import java.util.List;
diff --git a/modules/core/src/main/java/io/fluo/webindex/core/models/Page.java b/modules/core/src/main/java/webindex/core/models/Page.java
similarity index 98%
rename from modules/core/src/main/java/io/fluo/webindex/core/models/Page.java
rename to modules/core/src/main/java/webindex/core/models/Page.java
index e3e6632..6bfb6dd 100644
--- a/modules/core/src/main/java/io/fluo/webindex/core/models/Page.java
+++ b/modules/core/src/main/java/webindex/core/models/Page.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core.models;
+package webindex.core.models;
import java.io.Serializable;
import java.util.Objects;
diff --git a/modules/core/src/main/java/io/fluo/webindex/core/models/Pages.java b/modules/core/src/main/java/webindex/core/models/Pages.java
similarity index 97%
rename from modules/core/src/main/java/io/fluo/webindex/core/models/Pages.java
rename to modules/core/src/main/java/webindex/core/models/Pages.java
index 4f90805..de06db3 100644
--- a/modules/core/src/main/java/io/fluo/webindex/core/models/Pages.java
+++ b/modules/core/src/main/java/webindex/core/models/Pages.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core.models;
+package webindex.core.models;
import java.util.ArrayList;
import java.util.List;
diff --git a/modules/core/src/main/java/io/fluo/webindex/core/models/TopResults.java b/modules/core/src/main/java/webindex/core/models/TopResults.java
similarity index 97%
rename from modules/core/src/main/java/io/fluo/webindex/core/models/TopResults.java
rename to modules/core/src/main/java/webindex/core/models/TopResults.java
index 1d0ec54..6171b2e 100644
--- a/modules/core/src/main/java/io/fluo/webindex/core/models/TopResults.java
+++ b/modules/core/src/main/java/webindex/core/models/TopResults.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core.models;
+package webindex.core.models;
import java.util.ArrayList;
import java.util.List;
diff --git a/modules/core/src/main/java/io/fluo/webindex/core/models/URL.java b/modules/core/src/main/java/webindex/core/models/URL.java
similarity index 99%
rename from modules/core/src/main/java/io/fluo/webindex/core/models/URL.java
rename to modules/core/src/main/java/webindex/core/models/URL.java
index 31b0b65..04fb3f9 100644
--- a/modules/core/src/main/java/io/fluo/webindex/core/models/URL.java
+++ b/modules/core/src/main/java/webindex/core/models/URL.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core.models;
+package webindex.core.models;
import java.io.Serializable;
import java.util.Objects;
diff --git a/modules/core/src/test/java/io/fluo/webindex/core/DataConfigTest.java b/modules/core/src/test/java/webindex/core/DataConfigTest.java
similarity index 96%
rename from modules/core/src/test/java/io/fluo/webindex/core/DataConfigTest.java
rename to modules/core/src/test/java/webindex/core/DataConfigTest.java
index 4d7f76b..a92bcbd 100644
--- a/modules/core/src/test/java/io/fluo/webindex/core/DataConfigTest.java
+++ b/modules/core/src/test/java/webindex/core/DataConfigTest.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core;
+package webindex.core;
import org.junit.Assert;
import org.junit.Test;
diff --git a/modules/core/src/test/java/io/fluo/webindex/core/models/LinkTest.java b/modules/core/src/test/java/webindex/core/models/LinkTest.java
similarity index 96%
rename from modules/core/src/test/java/io/fluo/webindex/core/models/LinkTest.java
rename to modules/core/src/test/java/webindex/core/models/LinkTest.java
index 6fdb5f5..59cb7c2 100644
--- a/modules/core/src/test/java/io/fluo/webindex/core/models/LinkTest.java
+++ b/modules/core/src/test/java/webindex/core/models/LinkTest.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core.models;
+package webindex.core.models;
import org.junit.Assert;
import org.junit.Test;
diff --git a/modules/core/src/test/java/io/fluo/webindex/core/models/PageTest.java b/modules/core/src/test/java/webindex/core/models/PageTest.java
similarity index 97%
rename from modules/core/src/test/java/io/fluo/webindex/core/models/PageTest.java
rename to modules/core/src/test/java/webindex/core/models/PageTest.java
index cd59dbf..b252c3d 100644
--- a/modules/core/src/test/java/io/fluo/webindex/core/models/PageTest.java
+++ b/modules/core/src/test/java/webindex/core/models/PageTest.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core.models;
+package webindex.core.models;
import com.google.gson.Gson;
import org.junit.Assert;
diff --git a/modules/core/src/test/java/io/fluo/webindex/core/models/URLTest.java b/modules/core/src/test/java/webindex/core/models/URLTest.java
similarity index 99%
rename from modules/core/src/test/java/io/fluo/webindex/core/models/URLTest.java
rename to modules/core/src/test/java/webindex/core/models/URLTest.java
index 4cb6336..89f5b61 100644
--- a/modules/core/src/test/java/io/fluo/webindex/core/models/URLTest.java
+++ b/modules/core/src/test/java/webindex/core/models/URLTest.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.core.models;
+package webindex.core.models;
import java.text.ParseException;
diff --git a/modules/core/src/test/resources/log4j.properties b/modules/core/src/test/resources/log4j.properties
index f871402..ee43eea 100644
--- a/modules/core/src/test/resources/log4j.properties
+++ b/modules/core/src/test/resources/log4j.properties
@@ -17,5 +17,5 @@
log4j.appender.CA.layout=org.apache.log4j.PatternLayout
log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c] %-5p: %m%n
-log4j.logger.io.fluo=INFO
+log4j.logger.webindex=WARN
log4j.logger.Remoting=WARN
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/CalcSplits.java b/modules/data/src/main/java/webindex/data/CalcSplits.java
similarity index 87%
rename from modules/data/src/main/java/io/fluo/webindex/data/CalcSplits.java
rename to modules/data/src/main/java/webindex/data/CalcSplits.java
index 860b73a..422f83e 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/CalcSplits.java
+++ b/modules/data/src/main/java/webindex/data/CalcSplits.java
@@ -12,16 +12,10 @@
* the License.
*/
-package io.fluo.webindex.data;
+package webindex.data;
import java.util.SortedSet;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.data.fluo.UriMap.UriInfo;
-import io.fluo.webindex.data.spark.IndexEnv;
-import io.fluo.webindex.data.spark.IndexStats;
-import io.fluo.webindex.data.spark.IndexUtil;
-import io.fluo.webindex.data.util.WARCFileInputFormat;
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.RowColumn;
import org.apache.hadoop.conf.Configuration;
@@ -33,6 +27,12 @@
import org.archive.io.ArchiveReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.models.Page;
+import webindex.data.fluo.UriMap.UriInfo;
+import webindex.data.spark.IndexEnv;
+import webindex.data.spark.IndexStats;
+import webindex.data.spark.IndexUtil;
+import webindex.data.util.WARCFileInputFormat;
public class CalcSplits {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/Configure.java b/modules/data/src/main/java/webindex/data/Configure.java
similarity index 93%
rename from modules/data/src/main/java/io/fluo/webindex/data/Configure.java
rename to modules/data/src/main/java/webindex/data/Configure.java
index ff0872a..0dd0789 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/Configure.java
+++ b/modules/data/src/main/java/webindex/data/Configure.java
@@ -12,18 +12,18 @@
* the License.
*/
-package io.fluo.webindex.data;
+package webindex.data;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.Iterator;
-import io.fluo.webindex.core.DataConfig;
-import io.fluo.webindex.data.spark.IndexEnv;
import org.apache.fluo.api.config.FluoConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.DataConfig;
+import webindex.data.spark.IndexEnv;
public class Configure {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/Copy.java b/modules/data/src/main/java/webindex/data/Copy.java
similarity index 96%
rename from modules/data/src/main/java/io/fluo/webindex/data/Copy.java
rename to modules/data/src/main/java/webindex/data/Copy.java
index 957fde7..7976feb 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/Copy.java
+++ b/modules/data/src/main/java/webindex/data/Copy.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data;
+package webindex.data;
import java.io.BufferedInputStream;
import java.io.IOException;
@@ -20,8 +20,6 @@
import java.net.URL;
import java.util.List;
-import io.fluo.webindex.core.DataConfig;
-import io.fluo.webindex.data.spark.IndexEnv;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -30,6 +28,8 @@
import org.apache.spark.api.java.JavaSparkContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.DataConfig;
+import webindex.data.spark.IndexEnv;
public class Copy {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/FluoApp.java b/modules/data/src/main/java/webindex/data/FluoApp.java
similarity index 88%
rename from modules/data/src/main/java/io/fluo/webindex/data/FluoApp.java
rename to modules/data/src/main/java/webindex/data/FluoApp.java
index 0e6a9bf..182b863 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/FluoApp.java
+++ b/modules/data/src/main/java/webindex/data/FluoApp.java
@@ -12,12 +12,8 @@
* the License.
*/
-package io.fluo.webindex.data;
+package webindex.data;
-import io.fluo.webindex.data.fluo.DomainMap;
-import io.fluo.webindex.data.fluo.PageObserver;
-import io.fluo.webindex.data.fluo.UriMap;
-import io.fluo.webindex.serialization.WebindexKryoFactory;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.config.ObserverConfiguration;
import org.apache.fluo.recipes.accumulo.export.AccumuloExport;
@@ -25,6 +21,10 @@
import org.apache.fluo.recipes.accumulo.export.TableInfo;
import org.apache.fluo.recipes.core.export.ExportQueue;
import org.apache.fluo.recipes.kryo.KryoSimplerSerializer;
+import webindex.data.fluo.DomainMap;
+import webindex.data.fluo.PageObserver;
+import webindex.data.fluo.UriMap;
+import webindex.serialization.WebindexKryoFactory;
public class FluoApp {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/Init.java b/modules/data/src/main/java/webindex/data/Init.java
similarity index 87%
rename from modules/data/src/main/java/io/fluo/webindex/data/Init.java
rename to modules/data/src/main/java/webindex/data/Init.java
index 8812b69..1caa164 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/Init.java
+++ b/modules/data/src/main/java/webindex/data/Init.java
@@ -12,14 +12,8 @@
* the License.
*/
-package io.fluo.webindex.data;
+package webindex.data;
-import io.fluo.webindex.core.DataConfig;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.data.spark.IndexEnv;
-import io.fluo.webindex.data.spark.IndexStats;
-import io.fluo.webindex.data.spark.IndexUtil;
-import io.fluo.webindex.data.util.WARCFileInputFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.spark.SparkConf;
@@ -29,6 +23,12 @@
import org.archive.io.ArchiveReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.DataConfig;
+import webindex.core.models.Page;
+import webindex.data.spark.IndexEnv;
+import webindex.data.spark.IndexStats;
+import webindex.data.spark.IndexUtil;
+import webindex.data.util.WARCFileInputFormat;
public class Init {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/LoadHdfs.java b/modules/data/src/main/java/webindex/data/LoadHdfs.java
similarity index 93%
rename from modules/data/src/main/java/io/fluo/webindex/data/LoadHdfs.java
rename to modules/data/src/main/java/webindex/data/LoadHdfs.java
index 49dbd23..87d590b 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/LoadHdfs.java
+++ b/modules/data/src/main/java/webindex/data/LoadHdfs.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data;
+package webindex.data;
import java.io.File;
import java.io.IOException;
@@ -20,11 +20,6 @@
import java.util.List;
import com.google.common.util.concurrent.RateLimiter;
-import io.fluo.webindex.core.DataConfig;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.data.fluo.PageLoader;
-import io.fluo.webindex.data.spark.IndexEnv;
-import io.fluo.webindex.data.util.ArchiveUtil;
import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
import org.apache.fluo.api.client.LoaderExecutor;
@@ -42,6 +37,11 @@
import org.archive.io.warc.WARCReaderFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.DataConfig;
+import webindex.core.models.Page;
+import webindex.data.fluo.PageLoader;
+import webindex.data.spark.IndexEnv;
+import webindex.data.util.ArchiveUtil;
public class LoadHdfs {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/LoadS3.java b/modules/data/src/main/java/webindex/data/LoadS3.java
similarity index 92%
rename from modules/data/src/main/java/io/fluo/webindex/data/LoadS3.java
rename to modules/data/src/main/java/webindex/data/LoadS3.java
index fbb9b81..e16c69e 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/LoadS3.java
+++ b/modules/data/src/main/java/webindex/data/LoadS3.java
@@ -12,18 +12,13 @@
* the License.
*/
-package io.fluo.webindex.data;
+package webindex.data;
import java.io.File;
import java.net.URL;
import java.util.List;
import com.google.common.util.concurrent.RateLimiter;
-import io.fluo.webindex.core.DataConfig;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.data.fluo.PageLoader;
-import io.fluo.webindex.data.spark.IndexEnv;
-import io.fluo.webindex.data.util.ArchiveUtil;
import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
import org.apache.fluo.api.client.LoaderExecutor;
@@ -36,6 +31,11 @@
import org.archive.io.warc.WARCReaderFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.DataConfig;
+import webindex.core.models.Page;
+import webindex.data.fluo.PageLoader;
+import webindex.data.spark.IndexEnv;
+import webindex.data.util.ArchiveUtil;
public class LoadS3 {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/TestParser.java b/modules/data/src/main/java/webindex/data/TestParser.java
similarity index 93%
rename from modules/data/src/main/java/io/fluo/webindex/data/TestParser.java
rename to modules/data/src/main/java/webindex/data/TestParser.java
index e772c31..318db31 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/TestParser.java
+++ b/modules/data/src/main/java/webindex/data/TestParser.java
@@ -12,14 +12,11 @@
* the License.
*/
-package io.fluo.webindex.data;
+package webindex.data;
import java.net.URL;
import java.util.List;
-import io.fluo.webindex.core.DataConfig;
-import io.fluo.webindex.data.spark.IndexEnv;
-import io.fluo.webindex.data.util.ArchiveUtil;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
@@ -28,6 +25,9 @@
import org.archive.io.warc.WARCReaderFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.DataConfig;
+import webindex.data.spark.IndexEnv;
+import webindex.data.util.ArchiveUtil;
public class TestParser {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/fluo/DomainExport.java b/modules/data/src/main/java/webindex/data/fluo/DomainExport.java
similarity index 91%
rename from modules/data/src/main/java/io/fluo/webindex/data/fluo/DomainExport.java
rename to modules/data/src/main/java/webindex/data/fluo/DomainExport.java
index 6c93f26..5f87c6d 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/fluo/DomainExport.java
+++ b/modules/data/src/main/java/webindex/data/fluo/DomainExport.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.fluo;
+package webindex.data.fluo;
import java.util.Collections;
import java.util.Map;
@@ -21,8 +21,7 @@
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.RowColumn;
import org.apache.fluo.recipes.accumulo.export.DifferenceExport;
-
-import static io.fluo.webindex.data.util.FluoConstants.PAGECOUNT_COL;
+import webindex.data.util.FluoConstants;
public class DomainExport extends DifferenceExport<String, Long> {
@@ -37,7 +36,7 @@
if (count.orElse(0L) == 0) {
return Collections.emptyMap();
}
- return Collections.singletonMap(new RowColumn("d:" + domain, PAGECOUNT_COL),
+ return Collections.singletonMap(new RowColumn("d:" + domain, FluoConstants.PAGECOUNT_COL),
Bytes.of(count.get() + ""));
}
}
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/fluo/DomainMap.java b/modules/data/src/main/java/webindex/data/fluo/DomainMap.java
similarity index 97%
rename from modules/data/src/main/java/io/fluo/webindex/data/fluo/DomainMap.java
rename to modules/data/src/main/java/webindex/data/fluo/DomainMap.java
index bffdb3a..d3f12f0 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/fluo/DomainMap.java
+++ b/modules/data/src/main/java/webindex/data/fluo/DomainMap.java
@@ -12,12 +12,11 @@
* the License.
*/
-package io.fluo.webindex.data.fluo;
+package webindex.data.fluo;
import java.util.Iterator;
import java.util.Optional;
-import io.fluo.webindex.data.FluoApp;
import org.apache.fluo.api.client.TransactionBase;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.observer.Observer.Context;
@@ -28,6 +27,7 @@
import org.apache.fluo.recipes.core.map.Combiner;
import org.apache.fluo.recipes.core.map.Update;
import org.apache.fluo.recipes.core.map.UpdateObserver;
+import webindex.data.FluoApp;
public class DomainMap {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/fluo/PageExport.java b/modules/data/src/main/java/webindex/data/fluo/PageExport.java
similarity index 92%
rename from modules/data/src/main/java/io/fluo/webindex/data/fluo/PageExport.java
rename to modules/data/src/main/java/webindex/data/fluo/PageExport.java
index 341fb28..f3bf023 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/fluo/PageExport.java
+++ b/modules/data/src/main/java/webindex/data/fluo/PageExport.java
@@ -12,18 +12,18 @@
* the License.
*/
-package io.fluo.webindex.data.fluo;
+package webindex.data.fluo;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import com.google.common.collect.Sets.SetView;
-import io.fluo.webindex.core.Constants;
-import io.fluo.webindex.core.models.Link;
-import io.fluo.webindex.core.models.Page;
import org.apache.accumulo.core.data.Mutation;
import org.apache.fluo.recipes.accumulo.export.AccumuloExport;
+import webindex.core.Constants;
+import webindex.core.models.Link;
+import webindex.core.models.Page;
public class PageExport implements AccumuloExport<String> {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/fluo/PageLoader.java b/modules/data/src/main/java/webindex/data/fluo/PageLoader.java
similarity index 93%
rename from modules/data/src/main/java/io/fluo/webindex/data/fluo/PageLoader.java
rename to modules/data/src/main/java/webindex/data/fluo/PageLoader.java
index 797be9e..9b02deb 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/fluo/PageLoader.java
+++ b/modules/data/src/main/java/webindex/data/fluo/PageLoader.java
@@ -12,22 +12,22 @@
* the License.
*/
-package io.fluo.webindex.data.fluo;
+package webindex.data.fluo;
import java.net.MalformedURLException;
import java.util.Objects;
import com.google.common.base.Preconditions;
import com.google.gson.Gson;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.core.models.URL;
-import io.fluo.webindex.data.util.FluoConstants;
import org.apache.fluo.api.client.Loader;
import org.apache.fluo.api.client.TransactionBase;
import org.apache.fluo.recipes.core.data.RowHasher;
import org.apache.fluo.recipes.core.types.TypedTransactionBase;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.models.Page;
+import webindex.core.models.URL;
+import webindex.data.util.FluoConstants;
public class PageLoader implements Loader {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/fluo/PageObserver.java b/modules/data/src/main/java/webindex/data/fluo/PageObserver.java
similarity index 93%
rename from modules/data/src/main/java/io/fluo/webindex/data/fluo/PageObserver.java
rename to modules/data/src/main/java/webindex/data/fluo/PageObserver.java
index 282d4c0..55eeaff 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/fluo/PageObserver.java
+++ b/modules/data/src/main/java/webindex/data/fluo/PageObserver.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.fluo;
+package webindex.data.fluo;
import java.util.HashMap;
import java.util.Map;
@@ -20,11 +20,6 @@
import com.google.common.collect.Sets;
import com.google.gson.Gson;
-import io.fluo.webindex.core.models.Link;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.data.FluoApp;
-import io.fluo.webindex.data.fluo.UriMap.UriInfo;
-import io.fluo.webindex.data.util.FluoConstants;
import org.apache.fluo.api.client.TransactionBase;
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.Column;
@@ -37,6 +32,11 @@
import org.apache.fluo.recipes.core.types.TypedTransactionBase;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.models.Link;
+import webindex.core.models.Page;
+import webindex.data.FluoApp;
+import webindex.data.fluo.UriMap.UriInfo;
+import webindex.data.util.FluoConstants;
public class PageObserver extends AbstractObserver {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/fluo/UriCountExport.java b/modules/data/src/main/java/webindex/data/fluo/UriCountExport.java
similarity index 92%
rename from modules/data/src/main/java/io/fluo/webindex/data/fluo/UriCountExport.java
rename to modules/data/src/main/java/webindex/data/fluo/UriCountExport.java
index 2027ea1..9e2d616 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/fluo/UriCountExport.java
+++ b/modules/data/src/main/java/webindex/data/fluo/UriCountExport.java
@@ -12,17 +12,13 @@
* the License.
*/
-package io.fluo.webindex.data.fluo;
+package webindex.data.fluo;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
-import io.fluo.webindex.core.Constants;
-import io.fluo.webindex.core.models.URL;
-import io.fluo.webindex.data.fluo.UriMap.UriInfo;
-import io.fluo.webindex.data.util.FluoConstants;
import org.apache.accumulo.core.client.lexicoder.Lexicoder;
import org.apache.accumulo.core.client.lexicoder.ReverseLexicoder;
import org.apache.accumulo.core.client.lexicoder.ULongLexicoder;
@@ -31,6 +27,10 @@
import org.apache.fluo.api.data.Column;
import org.apache.fluo.api.data.RowColumn;
import org.apache.fluo.recipes.accumulo.export.DifferenceExport;
+import webindex.core.Constants;
+import webindex.core.models.URL;
+import webindex.data.fluo.UriMap.UriInfo;
+import webindex.data.util.FluoConstants;
public class UriCountExport extends DifferenceExport<String, UriInfo> {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/fluo/UriMap.java b/modules/data/src/main/java/webindex/data/fluo/UriMap.java
similarity index 97%
rename from modules/data/src/main/java/io/fluo/webindex/data/fluo/UriMap.java
rename to modules/data/src/main/java/webindex/data/fluo/UriMap.java
index d0cb4e8..1842507 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/fluo/UriMap.java
+++ b/modules/data/src/main/java/webindex/data/fluo/UriMap.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.fluo;
+package webindex.data.fluo;
import java.io.Serializable;
import java.util.HashMap;
@@ -21,8 +21,6 @@
import java.util.Optional;
import com.google.common.base.Preconditions;
-import io.fluo.webindex.core.models.URL;
-import io.fluo.webindex.data.FluoApp;
import org.apache.fluo.api.client.TransactionBase;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.observer.Observer.Context;
@@ -33,6 +31,8 @@
import org.apache.fluo.recipes.core.map.Combiner;
import org.apache.fluo.recipes.core.map.Update;
import org.apache.fluo.recipes.core.map.UpdateObserver;
+import webindex.core.models.URL;
+import webindex.data.FluoApp;
/**
* This class contains code related to a CollisionFreeMap that keeps track of the count of
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/spark/IndexEnv.java b/modules/data/src/main/java/webindex/data/spark/IndexEnv.java
similarity index 97%
rename from modules/data/src/main/java/io/fluo/webindex/data/spark/IndexEnv.java
rename to modules/data/src/main/java/webindex/data/spark/IndexEnv.java
index 72c351b..4ee9277 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/spark/IndexEnv.java
+++ b/modules/data/src/main/java/webindex/data/spark/IndexEnv.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.spark;
+package webindex.data.spark;
import java.io.BufferedReader;
import java.io.File;
@@ -29,11 +29,6 @@
import java.util.stream.Stream;
import com.google.common.base.Preconditions;
-import io.fluo.webindex.core.DataConfig;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.data.FluoApp;
-import io.fluo.webindex.data.fluo.PageObserver;
-import io.fluo.webindex.data.fluo.UriMap.UriInfo;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
@@ -59,6 +54,11 @@
import org.apache.spark.api.java.JavaSparkContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.DataConfig;
+import webindex.core.models.Page;
+import webindex.data.FluoApp;
+import webindex.data.fluo.PageObserver;
+import webindex.data.fluo.UriMap.UriInfo;
public class IndexEnv {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/spark/IndexStats.java b/modules/data/src/main/java/webindex/data/spark/IndexStats.java
similarity index 97%
rename from modules/data/src/main/java/io/fluo/webindex/data/spark/IndexStats.java
rename to modules/data/src/main/java/webindex/data/spark/IndexStats.java
index 7741ed6..0f73795 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/spark/IndexStats.java
+++ b/modules/data/src/main/java/webindex/data/spark/IndexStats.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.spark;
+package webindex.data.spark;
import java.io.Serializable;
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/spark/IndexUtil.java b/modules/data/src/main/java/webindex/data/spark/IndexUtil.java
similarity index 92%
rename from modules/data/src/main/java/io/fluo/webindex/data/spark/IndexUtil.java
rename to modules/data/src/main/java/webindex/data/spark/IndexUtil.java
index 26b29f5..077a55a 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/spark/IndexUtil.java
+++ b/modules/data/src/main/java/webindex/data/spark/IndexUtil.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.spark;
+package webindex.data.spark;
import java.util.ArrayList;
import java.util.List;
@@ -21,18 +21,6 @@
import java.util.TreeSet;
import com.google.gson.Gson;
-import io.fluo.webindex.core.Constants;
-import io.fluo.webindex.core.models.Link;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.core.models.URL;
-import io.fluo.webindex.data.fluo.DomainMap;
-import io.fluo.webindex.data.fluo.PageObserver;
-import io.fluo.webindex.data.fluo.UriCountExport;
-import io.fluo.webindex.data.fluo.UriMap;
-import io.fluo.webindex.data.fluo.UriMap.UriInfo;
-import io.fluo.webindex.data.util.ArchiveUtil;
-import io.fluo.webindex.data.util.FluoConstants;
-import io.fluo.webindex.serialization.WebindexKryoFactory;
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.Column;
import org.apache.fluo.api.data.RowColumn;
@@ -47,6 +35,18 @@
import org.archive.io.ArchiveReader;
import org.archive.io.ArchiveRecord;
import scala.Tuple2;
+import webindex.core.Constants;
+import webindex.core.models.Link;
+import webindex.core.models.Page;
+import webindex.core.models.URL;
+import webindex.data.fluo.DomainMap;
+import webindex.data.fluo.PageObserver;
+import webindex.data.fluo.UriCountExport;
+import webindex.data.fluo.UriMap;
+import webindex.data.fluo.UriMap.UriInfo;
+import webindex.data.util.ArchiveUtil;
+import webindex.data.util.FluoConstants;
+import webindex.serialization.WebindexKryoFactory;
public class IndexUtil {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/util/ArchiveUtil.java b/modules/data/src/main/java/webindex/data/util/ArchiveUtil.java
similarity index 96%
rename from modules/data/src/main/java/io/fluo/webindex/data/util/ArchiveUtil.java
rename to modules/data/src/main/java/webindex/data/util/ArchiveUtil.java
index 53aeda1..212103a 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/util/ArchiveUtil.java
+++ b/modules/data/src/main/java/webindex/data/util/ArchiveUtil.java
@@ -12,14 +12,11 @@
* the License.
*/
-package io.fluo.webindex.data.util;
+package webindex.data.util;
import java.io.IOException;
import java.text.ParseException;
-import io.fluo.webindex.core.models.Link;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.core.models.URL;
import org.apache.commons.io.IOUtils;
import org.archive.io.ArchiveRecord;
import org.json.JSONArray;
@@ -27,6 +24,9 @@
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.models.Link;
+import webindex.core.models.Page;
+import webindex.core.models.URL;
public class ArchiveUtil {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/util/DataUrl.java b/modules/data/src/main/java/webindex/data/util/DataUrl.java
similarity index 94%
rename from modules/data/src/main/java/io/fluo/webindex/data/util/DataUrl.java
rename to modules/data/src/main/java/webindex/data/util/DataUrl.java
index e8caa02..695b16f 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/util/DataUrl.java
+++ b/modules/data/src/main/java/webindex/data/util/DataUrl.java
@@ -12,11 +12,11 @@
* the License.
*/
-package io.fluo.webindex.data.util;
+package webindex.data.util;
import com.google.common.net.HostSpecifier;
import com.google.common.net.InternetDomainName;
-import io.fluo.webindex.core.models.URL;
+import webindex.core.models.URL;
public class DataUrl {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/util/FluoConstants.java b/modules/data/src/main/java/webindex/data/util/FluoConstants.java
similarity index 93%
rename from modules/data/src/main/java/io/fluo/webindex/data/util/FluoConstants.java
rename to modules/data/src/main/java/webindex/data/util/FluoConstants.java
index d30d447..23f56b3 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/util/FluoConstants.java
+++ b/modules/data/src/main/java/webindex/data/util/FluoConstants.java
@@ -12,12 +12,12 @@
* the License.
*/
-package io.fluo.webindex.data.util;
+package webindex.data.util;
-import io.fluo.webindex.core.Constants;
import org.apache.fluo.api.data.Column;
import org.apache.fluo.recipes.core.types.StringEncoder;
import org.apache.fluo.recipes.core.types.TypeLayer;
+import webindex.core.Constants;
public class FluoConstants {
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/util/WARCFileInputFormat.java b/modules/data/src/main/java/webindex/data/util/WARCFileInputFormat.java
similarity index 97%
rename from modules/data/src/main/java/io/fluo/webindex/data/util/WARCFileInputFormat.java
rename to modules/data/src/main/java/webindex/data/util/WARCFileInputFormat.java
index 1f9fd95..bdb63df 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/util/WARCFileInputFormat.java
+++ b/modules/data/src/main/java/webindex/data/util/WARCFileInputFormat.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.util;
+package webindex.data.util;
import java.io.IOException;
diff --git a/modules/data/src/main/java/io/fluo/webindex/data/util/WARCFileRecordReader.java b/modules/data/src/main/java/webindex/data/util/WARCFileRecordReader.java
similarity index 98%
rename from modules/data/src/main/java/io/fluo/webindex/data/util/WARCFileRecordReader.java
rename to modules/data/src/main/java/webindex/data/util/WARCFileRecordReader.java
index 5ce900b..994b304 100644
--- a/modules/data/src/main/java/io/fluo/webindex/data/util/WARCFileRecordReader.java
+++ b/modules/data/src/main/java/webindex/data/util/WARCFileRecordReader.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.util;
+package webindex.data.util;
import java.io.IOException;
diff --git a/modules/data/src/main/java/io/fluo/webindex/serialization/WebindexKryoFactory.java b/modules/data/src/main/java/webindex/serialization/WebindexKryoFactory.java
similarity index 84%
rename from modules/data/src/main/java/io/fluo/webindex/serialization/WebindexKryoFactory.java
rename to modules/data/src/main/java/webindex/serialization/WebindexKryoFactory.java
index 75de8d5..c3e5035 100644
--- a/modules/data/src/main/java/io/fluo/webindex/serialization/WebindexKryoFactory.java
+++ b/modules/data/src/main/java/webindex/serialization/WebindexKryoFactory.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.serialization;
+package webindex.serialization;
import java.io.Serializable;
import java.util.ArrayList;
@@ -20,11 +20,11 @@
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.pool.KryoFactory;
-import io.fluo.webindex.core.models.Link;
-import io.fluo.webindex.data.fluo.DomainExport;
-import io.fluo.webindex.data.fluo.PageExport;
-import io.fluo.webindex.data.fluo.UriCountExport;
-import io.fluo.webindex.data.fluo.UriMap.UriInfo;
+import webindex.core.models.Link;
+import webindex.data.fluo.DomainExport;
+import webindex.data.fluo.PageExport;
+import webindex.data.fluo.UriCountExport;
+import webindex.data.fluo.UriMap.UriInfo;
public class WebindexKryoFactory implements KryoFactory, Serializable {
diff --git a/modules/data/src/test/java/io/fluo/webindex/data/SparkTestUtil.java b/modules/data/src/test/java/webindex/data/SparkTestUtil.java
similarity index 96%
rename from modules/data/src/test/java/io/fluo/webindex/data/SparkTestUtil.java
rename to modules/data/src/test/java/webindex/data/SparkTestUtil.java
index 85e0f2f..24ebf4a 100644
--- a/modules/data/src/test/java/io/fluo/webindex/data/SparkTestUtil.java
+++ b/modules/data/src/test/java/webindex/data/SparkTestUtil.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data;
+package webindex.data;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
diff --git a/modules/data/src/test/java/io/fluo/webindex/data/fluo/it/IndexIT.java b/modules/data/src/test/java/webindex/data/fluo/it/IndexIT.java
similarity index 93%
rename from modules/data/src/test/java/io/fluo/webindex/data/fluo/it/IndexIT.java
rename to modules/data/src/test/java/webindex/data/fluo/it/IndexIT.java
index 9b83650..7c64004 100644
--- a/modules/data/src/test/java/io/fluo/webindex/data/fluo/it/IndexIT.java
+++ b/modules/data/src/test/java/webindex/data/fluo/it/IndexIT.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.fluo.it;
+package webindex.data.fluo.it;
import java.io.File;
import java.util.ArrayList;
@@ -22,18 +22,6 @@
import java.util.Map;
import com.google.common.collect.Lists;
-import io.fluo.webindex.core.models.Link;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.core.models.URL;
-import io.fluo.webindex.data.SparkTestUtil;
-import io.fluo.webindex.data.fluo.PageLoader;
-import io.fluo.webindex.data.fluo.UriMap.UriInfo;
-import io.fluo.webindex.data.spark.Hex;
-import io.fluo.webindex.data.spark.IndexEnv;
-import io.fluo.webindex.data.spark.IndexStats;
-import io.fluo.webindex.data.spark.IndexUtil;
-import io.fluo.webindex.data.util.ArchiveUtil;
-import io.fluo.webindex.data.util.DataUrl;
import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
import org.apache.fluo.api.client.LoaderExecutor;
@@ -55,6 +43,18 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Tuple2;
+import webindex.core.models.Link;
+import webindex.core.models.Page;
+import webindex.core.models.URL;
+import webindex.data.SparkTestUtil;
+import webindex.data.fluo.PageLoader;
+import webindex.data.fluo.UriMap.UriInfo;
+import webindex.data.spark.Hex;
+import webindex.data.spark.IndexEnv;
+import webindex.data.spark.IndexStats;
+import webindex.data.spark.IndexUtil;
+import webindex.data.util.ArchiveUtil;
+import webindex.data.util.DataUrl;
public class IndexIT extends AccumuloExportITBase {
diff --git a/modules/data/src/test/java/io/fluo/webindex/data/spark/Hex.java b/modules/data/src/test/java/webindex/data/spark/Hex.java
similarity index 98%
rename from modules/data/src/test/java/io/fluo/webindex/data/spark/Hex.java
rename to modules/data/src/test/java/webindex/data/spark/Hex.java
index 12b3c9f..2e47f29 100644
--- a/modules/data/src/test/java/io/fluo/webindex/data/spark/Hex.java
+++ b/modules/data/src/test/java/webindex/data/spark/Hex.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.spark;
+package webindex.data.spark;
import java.io.ByteArrayOutputStream;
diff --git a/modules/data/src/test/java/io/fluo/webindex/data/spark/IndexEnvTest.java b/modules/data/src/test/java/webindex/data/spark/IndexEnvTest.java
similarity index 96%
rename from modules/data/src/test/java/io/fluo/webindex/data/spark/IndexEnvTest.java
rename to modules/data/src/test/java/webindex/data/spark/IndexEnvTest.java
index df9d432..7d19d63 100644
--- a/modules/data/src/test/java/io/fluo/webindex/data/spark/IndexEnvTest.java
+++ b/modules/data/src/test/java/webindex/data/spark/IndexEnvTest.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.spark;
+package webindex.data.spark;
import java.util.SortedSet;
diff --git a/modules/data/src/test/java/io/fluo/webindex/data/spark/IndexUtilTest.java b/modules/data/src/test/java/webindex/data/spark/IndexUtilTest.java
similarity index 93%
rename from modules/data/src/test/java/io/fluo/webindex/data/spark/IndexUtilTest.java
rename to modules/data/src/test/java/webindex/data/spark/IndexUtilTest.java
index 3a326aa..fc72bb6 100644
--- a/modules/data/src/test/java/io/fluo/webindex/data/spark/IndexUtilTest.java
+++ b/modules/data/src/test/java/webindex/data/spark/IndexUtilTest.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.data.spark;
+package webindex.data.spark;
import java.io.BufferedReader;
import java.io.InputStream;
@@ -21,11 +21,6 @@
import java.util.Iterator;
import java.util.List;
-import io.fluo.webindex.core.models.Link;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.data.SparkTestUtil;
-import io.fluo.webindex.data.fluo.UriMap.UriInfo;
-import io.fluo.webindex.data.util.DataUrl;
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.RowColumn;
import org.apache.spark.api.java.JavaPairRDD;
@@ -36,6 +31,11 @@
import org.junit.Before;
import org.junit.Test;
import scala.Tuple2;
+import webindex.core.models.Link;
+import webindex.core.models.Page;
+import webindex.data.SparkTestUtil;
+import webindex.data.fluo.UriMap.UriInfo;
+import webindex.data.util.DataUrl;
public class IndexUtilTest {
diff --git a/modules/data/src/test/java/io/fluo/webindex/data/util/ArchiveUtilTest.java b/modules/data/src/test/java/webindex/data/util/ArchiveUtilTest.java
similarity index 96%
rename from modules/data/src/test/java/io/fluo/webindex/data/util/ArchiveUtilTest.java
rename to modules/data/src/test/java/webindex/data/util/ArchiveUtilTest.java
index 22f0dff..83b58e3 100644
--- a/modules/data/src/test/java/io/fluo/webindex/data/util/ArchiveUtilTest.java
+++ b/modules/data/src/test/java/webindex/data/util/ArchiveUtilTest.java
@@ -12,19 +12,19 @@
* the License.
*/
-package io.fluo.webindex.data.util;
+package webindex.data.util;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.util.Iterator;
-import io.fluo.webindex.core.models.Page;
import org.archive.io.ArchiveReader;
import org.archive.io.ArchiveRecord;
import org.archive.io.warc.WARCReaderFactory;
import org.junit.Assert;
import org.junit.Test;
+import webindex.core.models.Page;
public class ArchiveUtilTest {
diff --git a/modules/data/src/test/java/io/fluo/webindex/data/util/DataUrlTest.java b/modules/data/src/test/java/webindex/data/util/DataUrlTest.java
similarity index 98%
rename from modules/data/src/test/java/io/fluo/webindex/data/util/DataUrlTest.java
rename to modules/data/src/test/java/webindex/data/util/DataUrlTest.java
index d4ab9e8..d312b42 100644
--- a/modules/data/src/test/java/io/fluo/webindex/data/util/DataUrlTest.java
+++ b/modules/data/src/test/java/webindex/data/util/DataUrlTest.java
@@ -12,11 +12,11 @@
* the License.
*/
-package io.fluo.webindex.data.util;
+package webindex.data.util;
-import io.fluo.webindex.core.models.URL;
import org.junit.Assert;
import org.junit.Test;
+import webindex.core.models.URL;
public class DataUrlTest {
diff --git a/modules/ui/pom.xml b/modules/ui/pom.xml
index 5065393..167871c 100644
--- a/modules/ui/pom.xml
+++ b/modules/ui/pom.xml
@@ -139,7 +139,7 @@
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>io.fluo.webindex.ui.WebIndexApp</mainClass>
+ <mainClass>webindex.ui.WebIndexApp</mainClass>
</transformer>
</transformers>
</configuration>
diff --git a/modules/ui/src/main/java/io/fluo/webindex/ui/FluoHealthCheck.java b/modules/ui/src/main/java/webindex/ui/FluoHealthCheck.java
similarity index 96%
rename from modules/ui/src/main/java/io/fluo/webindex/ui/FluoHealthCheck.java
rename to modules/ui/src/main/java/webindex/ui/FluoHealthCheck.java
index b1a0a8c..2b63a44 100644
--- a/modules/ui/src/main/java/io/fluo/webindex/ui/FluoHealthCheck.java
+++ b/modules/ui/src/main/java/webindex/ui/FluoHealthCheck.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.ui;
+package webindex.ui;
import com.codahale.metrics.health.HealthCheck;
diff --git a/modules/ui/src/main/java/io/fluo/webindex/ui/WebIndexApp.java b/modules/ui/src/main/java/webindex/ui/WebIndexApp.java
similarity index 96%
rename from modules/ui/src/main/java/io/fluo/webindex/ui/WebIndexApp.java
rename to modules/ui/src/main/java/webindex/ui/WebIndexApp.java
index c1685f4..85d32e4 100644
--- a/modules/ui/src/main/java/io/fluo/webindex/ui/WebIndexApp.java
+++ b/modules/ui/src/main/java/webindex/ui/WebIndexApp.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.ui;
+package webindex.ui;
import java.io.File;
@@ -21,10 +21,10 @@
import io.dropwizard.setup.Bootstrap;
import io.dropwizard.setup.Environment;
import io.dropwizard.views.ViewBundle;
-import io.fluo.webindex.core.DataConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.core.util.AccumuloUtil;
+import webindex.core.DataConfig;
public class WebIndexApp extends Application<WebIndexConfig> {
diff --git a/modules/ui/src/main/java/io/fluo/webindex/ui/WebIndexConfig.java b/modules/ui/src/main/java/webindex/ui/WebIndexConfig.java
similarity index 91%
rename from modules/ui/src/main/java/io/fluo/webindex/ui/WebIndexConfig.java
rename to modules/ui/src/main/java/webindex/ui/WebIndexConfig.java
index 5347873..1328530 100644
--- a/modules/ui/src/main/java/io/fluo/webindex/ui/WebIndexConfig.java
+++ b/modules/ui/src/main/java/webindex/ui/WebIndexConfig.java
@@ -12,10 +12,10 @@
* the License.
*/
-package io.fluo.webindex.ui;
+package webindex.ui;
import io.dropwizard.Configuration;
-import io.fluo.webindex.core.DataConfig;
+import webindex.core.DataConfig;
public class WebIndexConfig extends Configuration {
diff --git a/modules/ui/src/main/java/io/fluo/webindex/ui/WebIndexResources.java b/modules/ui/src/main/java/webindex/ui/WebIndexResources.java
similarity index 92%
rename from modules/ui/src/main/java/io/fluo/webindex/ui/WebIndexResources.java
rename to modules/ui/src/main/java/webindex/ui/WebIndexResources.java
index 61c4bbd..879d883 100644
--- a/modules/ui/src/main/java/io/fluo/webindex/ui/WebIndexResources.java
+++ b/modules/ui/src/main/java/webindex/ui/WebIndexResources.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.ui;
+package webindex.ui;
import java.util.Iterator;
import java.util.Map;
@@ -26,22 +26,6 @@
import javax.ws.rs.core.MediaType;
import com.google.gson.Gson;
-import io.fluo.webindex.core.Constants;
-import io.fluo.webindex.core.DataConfig;
-import io.fluo.webindex.core.models.DomainStats;
-import io.fluo.webindex.core.models.Link;
-import io.fluo.webindex.core.models.Links;
-import io.fluo.webindex.core.models.Page;
-import io.fluo.webindex.core.models.Pages;
-import io.fluo.webindex.core.models.TopResults;
-import io.fluo.webindex.core.models.URL;
-import io.fluo.webindex.ui.util.Pager;
-import io.fluo.webindex.ui.util.WebUrl;
-import io.fluo.webindex.ui.views.HomeView;
-import io.fluo.webindex.ui.views.LinksView;
-import io.fluo.webindex.ui.views.PageView;
-import io.fluo.webindex.ui.views.PagesView;
-import io.fluo.webindex.ui.views.TopView;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
@@ -51,6 +35,22 @@
import org.apache.accumulo.core.security.Authorizations;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import webindex.core.Constants;
+import webindex.core.DataConfig;
+import webindex.core.models.DomainStats;
+import webindex.core.models.Link;
+import webindex.core.models.Links;
+import webindex.core.models.Page;
+import webindex.core.models.Pages;
+import webindex.core.models.TopResults;
+import webindex.core.models.URL;
+import webindex.ui.util.Pager;
+import webindex.ui.util.WebUrl;
+import webindex.ui.views.HomeView;
+import webindex.ui.views.LinksView;
+import webindex.ui.views.PageView;
+import webindex.ui.views.PagesView;
+import webindex.ui.views.TopView;
@Path("/")
public class WebIndexResources {
diff --git a/modules/ui/src/main/java/io/fluo/webindex/ui/util/Pager.java b/modules/ui/src/main/java/webindex/ui/util/Pager.java
similarity index 98%
rename from modules/ui/src/main/java/io/fluo/webindex/ui/util/Pager.java
rename to modules/ui/src/main/java/webindex/ui/util/Pager.java
index ec581d0..d69d95e 100644
--- a/modules/ui/src/main/java/io/fluo/webindex/ui/util/Pager.java
+++ b/modules/ui/src/main/java/webindex/ui/util/Pager.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.ui.util;
+package webindex.ui.util;
import java.util.Iterator;
import java.util.Map;
diff --git a/modules/ui/src/main/java/io/fluo/webindex/ui/util/WebUrl.java b/modules/ui/src/main/java/webindex/ui/util/WebUrl.java
similarity index 93%
rename from modules/ui/src/main/java/io/fluo/webindex/ui/util/WebUrl.java
rename to modules/ui/src/main/java/webindex/ui/util/WebUrl.java
index d520404..5f2e863 100644
--- a/modules/ui/src/main/java/io/fluo/webindex/ui/util/WebUrl.java
+++ b/modules/ui/src/main/java/webindex/ui/util/WebUrl.java
@@ -12,11 +12,11 @@
* the License.
*/
-package io.fluo.webindex.ui.util;
+package webindex.ui.util;
import com.google.common.net.HostSpecifier;
import com.google.common.net.InternetDomainName;
-import io.fluo.webindex.core.models.URL;
+import webindex.core.models.URL;
public class WebUrl {
diff --git a/modules/ui/src/main/java/io/fluo/webindex/ui/views/HomeView.java b/modules/ui/src/main/java/webindex/ui/views/HomeView.java
similarity index 95%
rename from modules/ui/src/main/java/io/fluo/webindex/ui/views/HomeView.java
rename to modules/ui/src/main/java/webindex/ui/views/HomeView.java
index 20a2ec6..dc2f1d4 100644
--- a/modules/ui/src/main/java/io/fluo/webindex/ui/views/HomeView.java
+++ b/modules/ui/src/main/java/webindex/ui/views/HomeView.java
@@ -12,7 +12,7 @@
* the License.
*/
-package io.fluo.webindex.ui.views;
+package webindex.ui.views;
import io.dropwizard.views.View;
diff --git a/modules/ui/src/main/java/io/fluo/webindex/ui/views/LinksView.java b/modules/ui/src/main/java/webindex/ui/views/LinksView.java
similarity index 91%
rename from modules/ui/src/main/java/io/fluo/webindex/ui/views/LinksView.java
rename to modules/ui/src/main/java/webindex/ui/views/LinksView.java
index 10d59a7..b168a32 100644
--- a/modules/ui/src/main/java/io/fluo/webindex/ui/views/LinksView.java
+++ b/modules/ui/src/main/java/webindex/ui/views/LinksView.java
@@ -12,10 +12,10 @@
* the License.
*/
-package io.fluo.webindex.ui.views;
+package webindex.ui.views;
import io.dropwizard.views.View;
-import io.fluo.webindex.core.models.Links;
+import webindex.core.models.Links;
public class LinksView extends View {
diff --git a/modules/ui/src/main/java/io/fluo/webindex/ui/views/PageView.java b/modules/ui/src/main/java/webindex/ui/views/PageView.java
similarity index 91%
rename from modules/ui/src/main/java/io/fluo/webindex/ui/views/PageView.java
rename to modules/ui/src/main/java/webindex/ui/views/PageView.java
index d76b72d..7b1a9fe 100644
--- a/modules/ui/src/main/java/io/fluo/webindex/ui/views/PageView.java
+++ b/modules/ui/src/main/java/webindex/ui/views/PageView.java
@@ -12,10 +12,10 @@
* the License.
*/
-package io.fluo.webindex.ui.views;
+package webindex.ui.views;
import io.dropwizard.views.View;
-import io.fluo.webindex.core.models.Page;
+import webindex.core.models.Page;
public class PageView extends View {
diff --git a/modules/ui/src/main/java/io/fluo/webindex/ui/views/PagesView.java b/modules/ui/src/main/java/webindex/ui/views/PagesView.java
similarity index 91%
rename from modules/ui/src/main/java/io/fluo/webindex/ui/views/PagesView.java
rename to modules/ui/src/main/java/webindex/ui/views/PagesView.java
index 941a58e..a856c63 100644
--- a/modules/ui/src/main/java/io/fluo/webindex/ui/views/PagesView.java
+++ b/modules/ui/src/main/java/webindex/ui/views/PagesView.java
@@ -12,10 +12,10 @@
* the License.
*/
-package io.fluo.webindex.ui.views;
+package webindex.ui.views;
import io.dropwizard.views.View;
-import io.fluo.webindex.core.models.Pages;
+import webindex.core.models.Pages;
public class PagesView extends View {
diff --git a/modules/ui/src/main/java/io/fluo/webindex/ui/views/TopView.java b/modules/ui/src/main/java/webindex/ui/views/TopView.java
similarity index 90%
rename from modules/ui/src/main/java/io/fluo/webindex/ui/views/TopView.java
rename to modules/ui/src/main/java/webindex/ui/views/TopView.java
index 26d35f2..b1927bf 100644
--- a/modules/ui/src/main/java/io/fluo/webindex/ui/views/TopView.java
+++ b/modules/ui/src/main/java/webindex/ui/views/TopView.java
@@ -12,10 +12,10 @@
* the License.
*/
-package io.fluo.webindex.ui.views;
+package webindex.ui.views;
import io.dropwizard.views.View;
-import io.fluo.webindex.core.models.TopResults;
+import webindex.core.models.TopResults;
public class TopView extends View {
diff --git a/modules/ui/src/main/resources/io/fluo/webindex/ui/views/common/footer.ftl b/modules/ui/src/main/resources/webindex/ui/views/common/footer.ftl
similarity index 100%
rename from modules/ui/src/main/resources/io/fluo/webindex/ui/views/common/footer.ftl
rename to modules/ui/src/main/resources/webindex/ui/views/common/footer.ftl
diff --git a/modules/ui/src/main/resources/io/fluo/webindex/ui/views/common/head.ftl b/modules/ui/src/main/resources/webindex/ui/views/common/head.ftl
similarity index 100%
rename from modules/ui/src/main/resources/io/fluo/webindex/ui/views/common/head.ftl
rename to modules/ui/src/main/resources/webindex/ui/views/common/head.ftl
diff --git a/modules/ui/src/main/resources/io/fluo/webindex/ui/views/common/header.ftl b/modules/ui/src/main/resources/webindex/ui/views/common/header.ftl
similarity index 100%
rename from modules/ui/src/main/resources/io/fluo/webindex/ui/views/common/header.ftl
rename to modules/ui/src/main/resources/webindex/ui/views/common/header.ftl
diff --git a/modules/ui/src/main/resources/io/fluo/webindex/ui/views/home.ftl b/modules/ui/src/main/resources/webindex/ui/views/home.ftl
similarity index 100%
rename from modules/ui/src/main/resources/io/fluo/webindex/ui/views/home.ftl
rename to modules/ui/src/main/resources/webindex/ui/views/home.ftl
diff --git a/modules/ui/src/main/resources/io/fluo/webindex/ui/views/links.ftl b/modules/ui/src/main/resources/webindex/ui/views/links.ftl
similarity index 100%
rename from modules/ui/src/main/resources/io/fluo/webindex/ui/views/links.ftl
rename to modules/ui/src/main/resources/webindex/ui/views/links.ftl
diff --git a/modules/ui/src/main/resources/io/fluo/webindex/ui/views/page.ftl b/modules/ui/src/main/resources/webindex/ui/views/page.ftl
similarity index 100%
rename from modules/ui/src/main/resources/io/fluo/webindex/ui/views/page.ftl
rename to modules/ui/src/main/resources/webindex/ui/views/page.ftl
diff --git a/modules/ui/src/main/resources/io/fluo/webindex/ui/views/pages.ftl b/modules/ui/src/main/resources/webindex/ui/views/pages.ftl
similarity index 100%
rename from modules/ui/src/main/resources/io/fluo/webindex/ui/views/pages.ftl
rename to modules/ui/src/main/resources/webindex/ui/views/pages.ftl
diff --git a/modules/ui/src/main/resources/io/fluo/webindex/ui/views/top.ftl b/modules/ui/src/main/resources/webindex/ui/views/top.ftl
similarity index 100%
rename from modules/ui/src/main/resources/io/fluo/webindex/ui/views/top.ftl
rename to modules/ui/src/main/resources/webindex/ui/views/top.ftl
diff --git a/pom.xml b/pom.xml
index eb100e9..79ee4e1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -229,8 +229,8 @@
<exclude>src/test/resources/*.warc</exclude>
<exclude>src/test/resources/data/set1/*.txt</exclude>
<exclude>src/main/resources/splits/*.txt</exclude>
- <exclude>src/main/resources/io/fluo/webindex/ui/views/*.ftl</exclude>
- <exclude>src/main/resources/io/fluo/webindex/ui/views/common/*.ftl</exclude>
+ <exclude>src/main/resources/webindex/ui/views/*.ftl</exclude>
+ <exclude>src/main/resources/webindex/ui/views/common/*.ftl</exclude>
<exclude>logs/*</exclude>
<exclude>paths/*</exclude>
<exclude>dependency-reduced-pom.xml</exclude>
@@ -242,8 +242,8 @@
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<systemPropertyVariables>
- <io.fluo.it.instance.name>it-instance-maven</io.fluo.it.instance.name>
- <io.fluo.it.instance.clear>false</io.fluo.it.instance.clear>
+ <fluo.it.instance.name>it-instance-maven</fluo.it.instance.name>
+ <fluo.it.instance.clear>false</fluo.it.instance.clear>
</systemPropertyVariables>
</configuration>
</plugin>
