SQOOP-2399: BigDecimalSplitter java.lang.ArrayIndexOutOfBoundsException
(Sowmya Ramesh via Venkat Ranganathan)
diff --git a/src/java/org/apache/sqoop/mapreduce/db/BigDecimalSplitter.java b/src/java/org/apache/sqoop/mapreduce/db/BigDecimalSplitter.java
index ebe6c40..a8db2a3 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/BigDecimalSplitter.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/BigDecimalSplitter.java
@@ -31,6 +31,7 @@
import com.cloudera.sqoop.config.ConfigurationHelper;
import com.cloudera.sqoop.mapreduce.db.DBSplitter;
import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat;
+import org.apache.sqoop.validation.ValidationException;
/**
* Implement DBSplitter over BigDecimal values.
@@ -39,7 +40,7 @@
private static final Log LOG = LogFactory.getLog(BigDecimalSplitter.class);
public List<InputSplit> split(Configuration conf, ResultSet results,
- String colName) throws SQLException {
+ String colName) throws SQLException, ValidationException {
BigDecimal minVal = results.getBigDecimal(1);
BigDecimal maxVal = results.getBigDecimal(2);
@@ -140,7 +141,12 @@
curVal = curVal.add(splitSize);
}
- if (splits.get(splits.size() - 1).compareTo(maxVal) != 0
+ /*
+ * If the sort order and collation of the char columns differ we can have
+ * a situation where minVal > maxVal and splits can be empty list.
+ */
+
+ if ((splits.size() > 1 && splits.get(splits.size() - 1).compareTo(maxVal) != 0)
|| splits.size() == 1) {
// We didn't end on the maxVal. Add that to the end of the list.
splits.add(maxVal);
diff --git a/src/java/org/apache/sqoop/mapreduce/db/DBSplitter.java b/src/java/org/apache/sqoop/mapreduce/db/DBSplitter.java
index b121d4b..9ea8caf 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/DBSplitter.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/DBSplitter.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.sqoop.validation.ValidationException;
/**
* DBSplitter will generate DBInputSplits to use with DataDrivenDBInputFormat.
@@ -40,5 +41,5 @@
* type), determine a set of splits that span the given values.
*/
List<InputSplit> split(Configuration conf, ResultSet results, String colName)
- throws SQLException;
+ throws SQLException, ValidationException;
}
diff --git a/src/java/org/apache/sqoop/mapreduce/db/DataDrivenDBInputFormat.java b/src/java/org/apache/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
index db96e41..136b30a 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
@@ -51,7 +51,7 @@
import com.cloudera.sqoop.mapreduce.db.FloatSplitter;
import com.cloudera.sqoop.mapreduce.db.IntegerSplitter;
import com.cloudera.sqoop.mapreduce.db.TextSplitter;
-import com.cloudera.sqoop.mapreduce.db.DBInputFormat.DBInputSplit;
+import org.apache.sqoop.validation.ValidationException;
/**
* A InputFormat that reads input data from an SQL table.
@@ -197,8 +197,12 @@
+ " type: " + sqlDataType);
}
- return splitter.split(job.getConfiguration(), results,
- getDBConf().getInputOrderBy());
+ try {
+ return splitter.split(job.getConfiguration(), results,
+ getDBConf().getInputOrderBy());
+ } catch (ValidationException e) {
+ throw new IOException(e);
+ }
} catch (SQLException e) {
throw new IOException(e);
} finally {
diff --git a/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java b/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
index d3085cd..9896d95 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
@@ -31,6 +31,7 @@
import com.cloudera.sqoop.config.ConfigurationHelper;
import com.cloudera.sqoop.mapreduce.db.BigDecimalSplitter;
import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat;
+import org.apache.sqoop.validation.ValidationException;
/**
* Implement DBSplitter over text strings.
@@ -59,7 +60,7 @@
* points, then map the resulting floating point values back into strings.
*/
public List<InputSplit> split(Configuration conf, ResultSet results,
- String colName) throws SQLException {
+ String colName) throws SQLException, ValidationException {
LOG.warn("Generating splits for a textual index column.");
LOG.warn("If your database sorts in a case-insensitive order, "
@@ -146,11 +147,16 @@
}
public List<String> split(int numSplits, String minString,
- String maxString, String commonPrefix) throws SQLException {
+ String maxString, String commonPrefix) throws SQLException, ValidationException {
BigDecimal minVal = stringToBigDecimal(minString);
BigDecimal maxVal = stringToBigDecimal(maxString);
+
+ if (minVal.compareTo(maxVal) > 0) {
+ throw new ValidationException( minVal + " is greater than " + maxVal);
+ }
+
List<BigDecimal> splitPoints = split(
new BigDecimal(numSplits), minVal, maxVal);
List<String> splitStrings = new ArrayList<String>();
diff --git a/src/test/org/apache/sqoop/mapreduce/db/TestTextSplitter.java b/src/test/org/apache/sqoop/mapreduce/db/TestTextSplitter.java
index 9c007d3..5cfb0a5 100644
--- a/src/test/org/apache/sqoop/mapreduce/db/TestTextSplitter.java
+++ b/src/test/org/apache/sqoop/mapreduce/db/TestTextSplitter.java
@@ -24,6 +24,8 @@
import com.cloudera.sqoop.mapreduce.db.TextSplitter;
import junit.framework.TestCase;
+import junit.framework.Test;
+import org.apache.sqoop.validation.ValidationException;
/**
* Test that the TextSplitter implementation creates a sane set of splits.
@@ -113,7 +115,7 @@
assertEquals("AVeryLon", out);
}
- public void testAlphabetSplit() throws SQLException {
+ public void testAlphabetSplit() throws SQLException, ValidationException {
// This should give us 25 splits, one per letter.
TextSplitter splitter = new TextSplitter();
List<String> splits = splitter.split(25, "A", "Z", "");
@@ -123,7 +125,18 @@
assertArrayEquals(expected, splits.toArray(new String [0]));
}
- public void testCommonPrefix() throws SQLException {
+ public void testAlphabetSplitWhenMinStringGreaterThanMaxString() throws SQLException {
+ TextSplitter splitter = new TextSplitter();
+ try {
+ splitter.split(4, "Z", "A", "");
+ fail();
+ } catch (ValidationException e) {
+ // expected
+ assertTrue(true);
+ }
+ }
+
+ public void testCommonPrefix() throws SQLException, ValidationException {
// Splits between 'Hand' and 'Hardy'
TextSplitter splitter = new TextSplitter();
List<String> splits = splitter.split(5, "nd", "rdy", "Ha");
