SQOOP-2294: Change to Avro schema name breaks some use cases
(Qian Xu via Jarek Jarcec Cecho)
diff --git a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java
index e70d23c..d5bfae2 100644
--- a/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/DataDrivenImportJob.java
@@ -23,8 +23,6 @@
import java.sql.SQLException;
import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericRecord;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -88,7 +86,8 @@
job.setOutputValueClass(NullWritable.class);
} else if (options.getFileLayout()
== SqoopOptions.FileLayout.AvroDataFile) {
- Schema schema = generateAvroSchema(tableName);
+ final String schemaNameOverride = null;
+ Schema schema = generateAvroSchema(tableName, schemaNameOverride);
try {
writeAvroSchema(schema);
} catch (final IOException e) {
@@ -99,9 +98,12 @@
} else if (options.getFileLayout()
== SqoopOptions.FileLayout.ParquetFile) {
Configuration conf = job.getConfiguration();
- // An Avro schema is required for creating a dataset that manages
- // Parquet data records. The import will fail, if schema is invalid.
- Schema schema = generateAvroSchema(tableName);
+ // Kite SDK requires an Avro schema to represent the data structure of
+ // target dataset. If the schema name equals to generated java class name,
+ // the import will fail. So we use table name as schema name and add a
+ // prefix "codegen_" to generated java class to avoid the conflict.
+ final String schemaNameOverride = tableName;
+ Schema schema = generateAvroSchema(tableName, schemaNameOverride);
String uri = getKiteUri(conf, tableName);
ParquetJob.configureImportJob(conf, schema, uri, options.isAppendMode(),
options.doHiveImport() && options.doOverwriteHiveTable());
@@ -123,11 +125,12 @@
}
}
- private Schema generateAvroSchema(String tableName) throws IOException {
+ private Schema generateAvroSchema(String tableName,
+ String schemaNameOverride) throws IOException {
ConnManager connManager = getContext().getConnManager();
AvroSchemaGenerator generator = new AvroSchemaGenerator(options,
connManager, tableName);
- return generator.generate();
+ return generator.generate(schemaNameOverride);
}
private void writeAvroSchema(final Schema schema) throws IOException {
diff --git a/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java b/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
index ed8e8b1..2576673 100644
--- a/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
+++ b/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
@@ -55,7 +55,7 @@
this.tableName = table;
}
- public Schema generate() throws IOException {
+ public Schema generate(String schemaNameOverride) throws IOException {
ClassWriter classWriter = new ClassWriter(options, connManager,
tableName, null);
Map<String, Integer> columnTypes = classWriter.getColumnTypes();
@@ -75,7 +75,8 @@
TableClassName tableClassName = new TableClassName(options);
String shortClassName = tableClassName.getShortClassForTable(tableName);
String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName);
- String avroName = "sqoop_import_" + (shortClassName == null ? avroTableName : shortClassName);
+ String avroName = schemaNameOverride != null ? schemaNameOverride :
+ (shortClassName == null ? avroTableName : shortClassName);
String avroNamespace = tableClassName.getPackageForTable();
String doc = "Sqoop import of " + avroTableName;
diff --git a/src/java/org/apache/sqoop/tool/CodeGenTool.java b/src/java/org/apache/sqoop/tool/CodeGenTool.java
index 6bd7f1d..ab339ad 100644
--- a/src/java/org/apache/sqoop/tool/CodeGenTool.java
+++ b/src/java/org/apache/sqoop/tool/CodeGenTool.java
@@ -90,6 +90,17 @@
return null;
}
LOG.info("Beginning code generation");
+
+ if (options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) {
+ String className = options.getClassName() != null ?
+ options.getClassName() : options.getTableName();
+ if (className.equalsIgnoreCase(options.getTableName())) {
+ className = "codegen_" + className;
+ options.setClassName(className);
+ LOG.info("Will generate java class as " + options.getClassName());
+ }
+ }
+
CompilationManager compileMgr = new CompilationManager(options);
ClassWriter classWriter = new ClassWriter(options, manager, tableName,
compileMgr);
