blob: 6068acafdeaeda892e0c2230fa0cd87c36961b87 [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.drill.exec.store.dfs;
import static java.util.Collections.unmodifiableMap;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.TableInstance;
import org.apache.drill.exec.store.table.function.TableParamDef;
import org.apache.drill.exec.store.table.function.TableSignature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* Describes the options for a format plugin
* extracted from the FormatPluginConfig subclass
*/
final class FormatPluginOptionsDescriptor {
private static final Logger logger = LoggerFactory.getLogger(FormatPluginOptionsDescriptor.class);
protected final Class<? extends FormatPluginConfig> pluginConfigClass;
protected final String typeName;
private final Map<String, TableParamDef> functionParamsByName;
/**
* Uses reflection to extract options based on the fields of the provided config class
* ("List extensions" field is ignored, pending removal, Char is turned into String)
* The class must be annotated with {@code @JsonTypeName("type name")}
* @param pluginConfigClass the config class we want to extract options from through reflection
*/
FormatPluginOptionsDescriptor(Class<? extends FormatPluginConfig> pluginConfigClass) {
this.pluginConfigClass = pluginConfigClass;
Map<String, TableParamDef> paramsByName = new LinkedHashMap<>();
Field[] fields = pluginConfigClass.getDeclaredFields();
// @JsonTypeName("text")
JsonTypeName annotation = pluginConfigClass.getAnnotation(JsonTypeName.class);
this.typeName = annotation != null ? annotation.value() : null;
if (this.typeName != null) {
paramsByName.put("type", TableParamDef.required("type", String.class, null));
}
for (Field field : fields) {
if (Modifier.isStatic(field.getModifiers())
// we want to deprecate this field
|| (field.getName().equals("extensions") && field.getType() == List.class)) {
continue;
}
Class<?> fieldType = field.getType();
if (fieldType == char.class) {
// calcite does not like char type. Just use String and enforce later that length == 1
fieldType = String.class;
}
paramsByName.put(field.getName(), TableParamDef.optional(field.getName(), fieldType, null));
}
this.functionParamsByName = unmodifiableMap(paramsByName);
}
public String getTypeName() { return typeName; }
/**
* Returns the table function signature for this format plugin config class.
*
* @param tableName the table for which we want a table function signature
* @param tableParameters common table parameters to be included
* @return the signature
*/
protected TableSignature getTableSignature(String tableName, List<TableParamDef> tableParameters) {
return TableSignature.of(tableName, tableParameters, params());
}
/**
* @return the parameters extracted from the provided format plugin config class
*/
private List<TableParamDef> params() {
return new ArrayList<>(functionParamsByName.values());
}
/**
* @return a readable String of the parameters and their names
*/
protected String presentParams() {
StringBuilder sb = new StringBuilder("(");
List<TableParamDef> params = params();
for (int i = 0; i < params.size(); i++) {
TableParamDef paramDef = params.get(i);
if (i != 0) {
sb.append(", ");
}
sb.append(paramDef.getName()).append(": ").append(paramDef.getType().getSimpleName());
}
sb.append(")");
return sb.toString();
}
/**
* Creates an instance of the FormatPluginConfig based on the passed parameters.
*
* @param t the signature and the parameters passed to the table function
* @param mapper
* @return the corresponding config
*/
FormatPluginConfig createConfigForTable(TableInstance t, ObjectMapper mapper, FormatPluginConfig baseConfig) {
ConfigCreator configCreator = new ConfigCreator(t, mapper, baseConfig);
return configCreator.createNewStyle();
}
@Override
public String toString() {
return "OptionsDescriptor [pluginConfigClass=" + pluginConfigClass + ", typeName=" + typeName
+ ", functionParamsByName=" + functionParamsByName + "]";
}
/**
* Implements a table function to specify a format config. Provides two
* Implementations. The first is the "legacy" version (Drill 1.17 and
* before), which relies on a niladic constructor and mutable fields.
* Since mutable fields conflicts with the desire for configs to be
* immutable, the newer version (Drill 1.8 and later) use JSON serialization
* to create a JSON object with the desired properties and to seriarialize
* that object to a config. Since Jackson allows creating JSON objects
* from an existing config, this newer method merges the existing plugin
* properties with those specified in the table function. Essentially
* the table function "inherits" any existing config, "overriding" only
* those properties which are specified. Prior to Drill 1.18, a table
* function inherited the default properties, even if there was an
* existing plugin for the target file. See DRILL-6168. The original
* behavior is retained in case we find we need to add an option to
* cause Drill to revert to the old behavior.
*/
private class ConfigCreator {
final TableInstance t;
final FormatPluginConfig baseConfig;
final List<TableParamDef> formatParams;
final List<Object> formatParamsValues;
final ObjectMapper mapper;
public ConfigCreator(TableInstance table, ObjectMapper mapper, FormatPluginConfig baseConfig) {
this.t = table;
this.mapper = mapper;
// Abundance of caution: if the base is not of the correct
// type, just ignore it to avoid introducing new errors.
// Drill prior to 1.18 didn't use a base config.
if (baseConfig == null || baseConfig.getClass() != pluginConfigClass) {
this.baseConfig = null;
} else {
this.baseConfig = baseConfig;
}
formatParams = t.sig.getSpecificParams();
// Exclude common params values, leave only format related params
formatParamsValues = t.params.subList(0, t.params.size() - t.sig.getCommonParams().size());
}
public FormatPluginConfig createNewStyle() {
verifyType();
ObjectNode configObject = makeConfigNode();
applyParams(configObject);
// Do the following to visualize the merged object
// System.out.println(mapper.writeValueAsString(configObject));
return nodeToConfig(configObject);
}
/**
* Create a JSON node for the config: from the existing config
* if available, else an empty node.
*/
private ObjectNode makeConfigNode() {
if (baseConfig == null) {
ObjectNode configObject = mapper.createObjectNode();
// Type field is required to deserialize config
configObject.replace("type",
mapper.convertValue(typeName, JsonNode.class));
return configObject;
} else {
return mapper.valueToTree(baseConfig);
}
}
/**
* Replace any existing properties with the fields from the
* table function.
*/
private void applyParams(ObjectNode configObject) {
for (int i = 1; i < formatParamsValues.size(); i++) {
applyParam(configObject, i);
}
}
private void applyParam(ObjectNode configObject, int i) {
Object param = paramValue(i);
// when null is passed, we leave the default defined in the config instance
if (param != null) {
configObject.replace(formatParams.get(i).getName(),
mapper.convertValue(param, JsonNode.class));
}
}
private Object paramValue(int i) {
Object param = formatParamsValues.get(i);
if (param != null && param instanceof String) {
// normalize Java literals, ex: \t, \n, \r
param = StringEscapeUtils.unescapeJava((String) param);
}
return param;
}
/**
* Convert the JSON node to a format config.
*/
private FormatPluginConfig nodeToConfig(ObjectNode configObject) {
try {
return mapper.readerFor(pluginConfigClass).readValue(configObject);
} catch (IOException e) {
String jsonConfig;
try {
jsonConfig = mapper.writeValueAsString(configObject);
} catch (JsonProcessingException e1) {
jsonConfig = "unavailable: " + e1.getMessage();
}
throw UserException.parseError(e)
.message(
"configuration for format of type %s can not be created (class: %s)",
typeName, pluginConfigClass.getName())
.addContext("table", t.sig.getName())
.addContext("JSON configuration", jsonConfig)
.build(logger);
}
}
/**
* Creates a format plugin config in the style prior to
* Drill 1.8: binds parameters to public, mutable fields.
* However, this causes issues: fields should be immutable (DRILL-7612, DRILL-6672).
* Also, this style does not allow retaining some fields
* while customizing others. (DRILL-6168).
* @return
*/
@SuppressWarnings("unused")
public FormatPluginConfig createOldStyle() {
verifyType();
FormatPluginConfig config = configInstance();
bindParams(config);
return config;
}
public void verifyType() {
// Per the constructor, the first param is always "type"
TableParamDef typeParamDef = formatParams.get(0);
Object typeParam = formatParamsValues.get(0);
if (!typeParamDef.getName().equals("type")
|| typeParamDef.getType() != String.class
|| !(typeParam instanceof String)
|| !typeName.equalsIgnoreCase((String)typeParam)) {
// if we reach here, there's a bug as all signatures generated start with a type parameter
throw UserException.parseError()
.message(
"This function signature is not supported: %s\n"
+ "expecting %s",
t.presentParams(), presentParams())
.addContext("table", t.sig.getName())
.build(logger);
}
}
public FormatPluginConfig configInstance() {
try {
return pluginConfigClass.newInstance();
} catch (InstantiationException | IllegalAccessException e) {
throw UserException.parseError(e)
.message(
"configuration for format of type %s can not be created (class: %s)",
typeName, pluginConfigClass.getName())
.addContext("table", t.sig.getName())
.build(logger);
}
}
private void bindParams(FormatPluginConfig config) {
for (int i = 1; i < formatParamsValues.size(); i++) {
bindParam(config, i);
}
}
private void bindParam(FormatPluginConfig config, int i) {
Object param = paramValue(i);
if (param == null) {
// when null is passed, we leave the default defined in the config class
return;
}
TableParamDef paramDef = formatParams.get(i);
TableParamDef expectedParamDef = functionParamsByName.get(paramDef.getName());
if (expectedParamDef == null || expectedParamDef.getType() != paramDef.getType()) {
throw UserException.parseError()
.message(
"The parameters provided are not applicable to the type specified:\n"
+ "provided: %s\nexpected: %s",
t.presentParams(), presentParams())
.addContext("table", t.sig.getName())
.build(logger);
}
try {
Field field = pluginConfigClass.getField(paramDef.getName());
field.setAccessible(true);
if (field.getType() == char.class && param instanceof String) {
String stringParam = (String) param;
if (stringParam.length() != 1) {
throw UserException.parseError()
.message("Expected single character but was String: %s", stringParam)
.addContext("Table", t.sig.getName())
.addContext("Parameter", paramDef.getName())
.build(logger);
}
param = stringParam.charAt(0);
}
field.set(config, param);
} catch (IllegalAccessException | NoSuchFieldException | SecurityException e) {
throw UserException.parseError(e)
.message("Can not set value %s to parameter %s: %s", param, paramDef.getName(), paramDef.getType())
.addContext("Table", t.sig.getName())
.addContext("Parameter", paramDef.getName())
.build(logger);
}
}
}
}