SQOOP-3332: Extend Documentation of --resilient flag and add warning message when detected
(Fero Szabo via Boglarka Egyed)
diff --git a/src/docs/user/connectors.txt b/src/docs/user/connectors.txt
index f1c7aeb..59e3e00 100644
--- a/src/docs/user/connectors.txt
+++ b/src/docs/user/connectors.txt
@@ -147,17 +147,32 @@
Resilient operations
^^^^^^^^^^^^^^^^^^^^
-You can override the default and use resilient operations during export.
+You can override the default and use resilient operations during export or import.
This will retry failed operations, i.e. if the connection gets dropped by
SQL Server, the mapper will try to reconnect and continue from where it was before.
-The split-by column has to be specified and it is also required to be unique
-and in ascending order.
-For example:
+In case of export, the +\--resilient+ option will ensure that Sqoop will try to recover
+from connection resets.
+
+In case of import, however, one has to use both the +\--resilient+ option and specify
+the +\--split-by+ column to trigger the retry mechanism. An important requirement is
+that the data must be unique and ordered ascending by the split-by column, otherwise
+records could be either lost or duplicated.
+
+Example commands using resilient operations:
----
$ sqoop export ... --export-dir custom_dir --table custom_table -- --resilient
----
+Importing from a table:
+----
+$ sqoop import ... --table custom_table --split-by id -- --resilient
+----
+
+Importing via a query:
+----
+$ sqoop import ... --query "SELECT ... WHERE $CONDITIONS" --split-by ordered_column -- --resilient
+
Schema support
^^^^^^^^^^^^^^
diff --git a/src/java/org/apache/sqoop/manager/SQLServerManager.java b/src/java/org/apache/sqoop/manager/SQLServerManager.java
index c98ad2d..33c57bc 100644
--- a/src/java/org/apache/sqoop/manager/SQLServerManager.java
+++ b/src/java/org/apache/sqoop/manager/SQLServerManager.java
@@ -333,6 +333,11 @@
this.tableHints = hints;
}
+ if (cmdLine.hasOption(SqlServerManagerContextConfigurator.RESILIENT_OPTION)) {
+ LOG.warn("Sqoop will use resilient operations! In case of import, " +
+ "the split-by column also has to be specified, unique, and in ascending order.");
+ }
+
identityInserts = cmdLine.hasOption(IDENTITY_INSERT);
}
@@ -359,6 +364,9 @@
.withDescription("Allow identity inserts")
.withLongOpt(IDENTITY_INSERT).create());
+ extraOptions.addOption(OptionBuilder
+ .withLongOpt(SqlServerManagerContextConfigurator.RESILIENT_OPTION).create());
+
return extraOptions;
}
diff --git a/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java b/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java
index cf58f63..eab8a42 100644
--- a/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java
+++ b/src/java/org/apache/sqoop/manager/SqlServerManagerContextConfigurator.java
@@ -28,7 +28,7 @@
public class SqlServerManagerContextConfigurator {
- private static final String RESILIENT_OPTION = "resilient";
+ public static final String RESILIENT_OPTION = "resilient";
/**
* Check if the user has requested the operation to be resilient.
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java
index fc1c489..79e37f0 100644
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerManagerImportTest.java
@@ -28,9 +28,11 @@
import org.apache.sqoop.manager.SQLServerManager;
import org.apache.sqoop.testutil.ArgumentArrayBuilder;
import org.apache.sqoop.testutil.ImportJobTestCase;
+import org.apache.sqoop.util.ExpectedLogMessage;
import org.apache.sqoop.util.FileListing;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -129,7 +131,7 @@
private final String tableName;
public SQLServerManagerImportTest(ArgumentArrayBuilder builder, String tableName) {
- this.builder = builder;
+ this.builder = new ArgumentArrayBuilder().with(builder);
this.tableName = tableName;
}
@@ -266,6 +268,9 @@
}
}
+ @Rule
+ public ExpectedLogMessage logMessage = new ExpectedLogMessage();
+
@Test
public void testImportSimple() throws IOException {
doImportAndVerify(builder, tableName);
@@ -285,6 +290,7 @@
@Test
public void testImportTableResilient() throws IOException {
+ logMessage.expectWarn("Sqoop will use resilient operations! In case of import, the split-by column also has to be specified, unique, and in ascending order.");
builder.withToolOption("resilient");
doImportAndVerify(builder, tableName);
}
