[CARBONDATA-2172][Lucene] Add text_columns property for Lucene DataMap Add text_columns property for Lucene DataMap This closes #2019
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java index 0223ae2..e7c72e8 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -163,7 +163,8 @@ } public TableDataMap registerDataMap(AbsoluteTableIdentifier identifier, - DataMapSchema dataMapSchema, DataMapFactory dataMapFactory) throws IOException { + DataMapSchema dataMapSchema, DataMapFactory dataMapFactory) + throws IOException, MalformedDataMapCommandException { String table = identifier.getCarbonTableIdentifier().getTableUniqueName(); // Just update the segmentRefreshMap with the table if not added. getTableSegmentRefresher(identifier);
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java index 7bf04c9..ef9bb66 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; +import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException; import org.apache.carbondata.core.datamap.DataMapDistributable; import org.apache.carbondata.core.datamap.DataMapLevel; import org.apache.carbondata.core.datamap.DataMapMeta; @@ -34,7 +35,8 @@ /** * Initialization of Datamap factory with the identifier and datamap name */ - void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) throws IOException; + void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) + throws IOException, MalformedDataMapCommandException; /** * Return a new write for this datamap
diff --git a/datamap/lucene/pom.xml b/datamap/lucene/pom.xml index ee504c6..4019065 100644 --- a/datamap/lucene/pom.xml +++ b/datamap/lucene/pom.xml
@@ -26,6 +26,11 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + <version>3.3.2</version> + </dependency> + <dependency> <groupId>org.apache.lucene</groupId> <artifactId>lucene-core</artifactId> <version>${lucene.version}</version>
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java index 5eb7054..3a1adab 100644 --- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java +++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
@@ -23,6 +23,7 @@ import java.util.Objects; import org.apache.carbondata.common.annotations.InterfaceAudience; +import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.datamap.DataMapDistributable; @@ -32,14 +33,15 @@ import org.apache.carbondata.core.datamap.dev.DataMapWriter; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.CarbonMetadata; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; -import org.apache.carbondata.core.metadata.schema.table.TableInfo; -import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.core.scan.filter.intf.ExpressionType; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.events.Event; +import org.apache.commons.lang.StringUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; @@ -49,6 +51,8 @@ @InterfaceAudience.Internal abstract class LuceneDataMapFactoryBase<T extends DataMap> implements DataMapFactory<T> { + static final String TEXT_COLUMNS = "text_columns"; + /** * Logger */ @@ -76,7 +80,7 @@ @Override public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) - throws IOException { + throws IOException, MalformedDataMapCommandException { Objects.requireNonNull(identifier); Objects.requireNonNull(dataMapSchema); @@ -97,34 +101,17 @@ throw new IOException(errorMessage); } - TableInfo tableInfo = carbonTable.getTableInfo(); - List<ColumnSchema> lstCoumnSchemas = tableInfo.getFactTable().getListOfColumns(); - - // currently add all columns into lucene indexer - // TODO:only add index columns - List<String> indexedColumns = new ArrayList<String>(); - for (ColumnSchema columnSchema : lstCoumnSchemas) { - if (!columnSchema.isInvisible()) { - indexedColumns.add(columnSchema.getColumnName()); - } - } - - // get indexed columns - // Map<String, String> properties = dataMapSchema.getProperties(); - // String columns = properties.get("text_column"); - // if (columns != null) { - // String[] columnArray = columns.split(CarbonCommonConstants.COMMA, -1); - // Collections.addAll(indexedColumns, columnArray); - // } + // validate DataMapSchema and get index columns + List<String> indexedColumns = validateAndGetIndexedColumns(dataMapSchema, carbonTable); // add optimizedOperations List<ExpressionType> optimizedOperations = new ArrayList<ExpressionType>(); - // optimizedOperations.add(ExpressionType.EQUALS); - // optimizedOperations.add(ExpressionType.GREATERTHAN); - // optimizedOperations.add(ExpressionType.GREATERTHAN_EQUALTO); - // optimizedOperations.add(ExpressionType.LESSTHAN); - // optimizedOperations.add(ExpressionType.LESSTHAN_EQUALTO); - // optimizedOperations.add(ExpressionType.NOT); + // optimizedOperations.add(ExpressionType.EQUALS); + // optimizedOperations.add(ExpressionType.GREATERTHAN); + // optimizedOperations.add(ExpressionType.GREATERTHAN_EQUALTO); + // optimizedOperations.add(ExpressionType.LESSTHAN); + // optimizedOperations.add(ExpressionType.LESSTHAN_EQUALTO); + // optimizedOperations.add(ExpressionType.NOT); optimizedOperations.add(ExpressionType.TEXT_MATCH); this.dataMapMeta = new DataMapMeta(indexedColumns, optimizedOperations); @@ -134,6 +121,52 @@ } /** + * validate Lucene DataMap + * 1. require TEXT_COLUMNS property + * 2. TEXT_COLUMNS can't contains illegal argument(empty, blank) + * 3. TEXT_COLUMNS can't contains duplicate same columns + * 4. TEXT_COLUMNS should be exists in table columns + * 5. TEXT_COLUMNS support only String DataType columns + */ + private List<String> validateAndGetIndexedColumns(DataMapSchema dataMapSchema, + CarbonTable carbonTable) throws MalformedDataMapCommandException { + String textColumnsStr = dataMapSchema.getProperties().get(TEXT_COLUMNS); + if (textColumnsStr == null || StringUtils.isBlank(textColumnsStr)) { + throw new MalformedDataMapCommandException( + "Lucene DataMap require proper TEXT_COLUMNS property."); + } + String[] textColumns = textColumnsStr.split(",", -1); + for (int i = 0; i < textColumns.length; i++) { + textColumns[i] = textColumns[i].trim().toLowerCase(); + } + for (int i = 0; i < textColumns.length; i++) { + if (textColumns[i].isEmpty()) { + throw new MalformedDataMapCommandException("TEXT_COLUMNS contains illegal argument."); + } + for (int j = i + 1; j < textColumns.length; j++) { + if (textColumns[i].equals(textColumns[j])) { + throw new MalformedDataMapCommandException( + "TEXT_COLUMNS has duplicate columns :" + textColumns[i]); + } + } + } + List<String> textColumnList = new ArrayList<String>(textColumns.length); + for (int i = 0; i < textColumns.length; i++) { + CarbonColumn column = carbonTable.getColumnByName(carbonTable.getTableName(), textColumns[i]); + if (null == column) { + throw new MalformedDataMapCommandException("TEXT_COLUMNS: " + textColumns[i] + + " does not exist in table. Please check create DataMap statement."); + } else if (column.getDataType() != DataTypes.STRING) { + throw new MalformedDataMapCommandException( + "TEXT_COLUMNS only supports String column. " + "Unsupported column: " + textColumns[i] + + ", DataType: " + column.getDataType()); + } + textColumnList.add(column.getColName()); + } + return textColumnList; + } + + /** * Return a new write for this datamap */ public DataMapWriter createWriter(String segmentId, String writeDirectoryPath) {
diff --git a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala index 5e28e8a..bfcfa67 100644 --- a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala +++ b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll -import org.apache.carbondata.core.metadata.CarbonMetadata +import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll { @@ -42,9 +42,7 @@ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT') """.stripMargin) sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')") - } - test("test lucene fine grain data map") { sql("DROP TABLE IF EXISTS datamap_test") sql( """ @@ -52,11 +50,65 @@ | STORED BY 'carbondata' | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT') """.stripMargin) + } + test("validate TEXT_COLUMNS DataMap property") { + // require TEXT_COLUMNS + var exception = intercept[MalformedDataMapCommandException](sql( + s""" + | CREATE DATAMAP dm1 ON TABLE datamap_test + | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory' + """.stripMargin)) + + assertResult("Lucene DataMap require proper TEXT_COLUMNS property.")(exception.getMessage) + + // illegal argumnet. + exception = intercept[MalformedDataMapCommandException](sql( + s""" + | CREATE DATAMAP dm1 ON TABLE datamap_test + | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory' + | DMProperties('text_COLUMNS'='name, ') + """.stripMargin)) + + assertResult("TEXT_COLUMNS contains illegal argument.")(exception.getMessage) + + // not exists + exception = intercept[MalformedDataMapCommandException](sql( + s""" + | CREATE DATAMAP dm1 ON TABLE datamap_test + | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory' + | DMProperties('text_COLUMNS'='city,school') + """.stripMargin)) + + assertResult("TEXT_COLUMNS: school does not exist in table. Please check create DataMap statement.")(exception.getMessage) + + // duplicate columns + exception = intercept[MalformedDataMapCommandException](sql( + s""" + | CREATE DATAMAP dm1 ON TABLE datamap_test + | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory' + | DMProperties('text_COLUMNS'='name,city,name') + """.stripMargin)) + + assertResult("TEXT_COLUMNS has duplicate columns :name")(exception.getMessage) + + // only support String DataType + exception = intercept[MalformedDataMapCommandException](sql( + s""" + | CREATE DATAMAP dm1 ON TABLE datamap_test + | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory' + | DMProperties('text_COLUMNS'='city,id') + """.stripMargin)) + + assertResult("TEXT_COLUMNS only supports String column. Unsupported column: id, DataType: INT")(exception.getMessage) + } + + test("test lucene fine grain data map") { sql( s""" | CREATE DATAMAP dm ON TABLE datamap_test | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory' + | DMProperties('TEXT_COLUMNS'='Name , cIty') """.stripMargin) sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")