DRILL-8415: Upgrade Jackson 2.14.3 → 2.16.1 (#2866)
* Upgrade Jackson 2.14.3 → 2.16.1.
* Do some list sorting in java-exec's drill-module.conf.
* Bump JDBC jar size to 54.5Mb.
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index df311bf..c496b9e 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -19,6 +19,8 @@
# This file tells Drill to consider this module when class path scanning.
# This file can also include any supplementary configuration information.
# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+# Please keep the lists that follow sorted (while respecting existing
+# groupings and comment lines).
drill {
classpath.scanning {
@@ -42,13 +44,13 @@
],
packages: ${?drill.classpath.scanning.packages} [
+ org.apache.drill.exec.coord.zk,
org.apache.drill.exec.expr,
org.apache.drill.exec.physical,
- org.apache.drill.exec.store,
- org.apache.drill.exec.rpc.user.security,
org.apache.drill.exec.rpc.security,
+ org.apache.drill.exec.rpc.user.security,
org.apache.drill.exec.server.rest.auth,
- org.apache.drill.exec.coord.zk,
+ org.apache.drill.exec.store,
org.apache.drill.exec.store.security
],
@@ -542,70 +544,82 @@
# enabled by setting this flag to true. By default it's set to false such that
# query will fail if there is not enough memory
drill.exec.hashagg.fallback.enabled: false,
+ drill.exec.http.rest.errors.verbose: false,
+ drill.exec.implicit_casts.joins.enabled: false,
+ drill.exec.memory.operator.output_batch_size : 16777216, # 16 MB
+ drill.exec.memory.operator.output_batch_size_avail_mem_factor : 0.1,
+ drill.exec.stats.logging.batch_size: false,
+ drill.exec.stats.logging.fine_grained.batch_size: false,
+ drill.exec.stats.logging.enabled_operators: all,
drill.exec.storage.file.partition.column.label: "dir",
drill.exec.storage.implicit.filename.column.label: "filename",
drill.exec.storage.implicit.filepath.column.label: "filepath",
drill.exec.storage.implicit.fqn.column.label: "fqn",
- drill.exec.storage.implicit.suffix.column.label: "suffix",
- drill.exec.storage.implicit.row_group_index.column.label: "rgi",
- drill.exec.storage.implicit.row_group_start.column.label: "rgs",
- drill.exec.storage.implicit.row_group_length.column.label: "rgl",
drill.exec.storage.implicit.last_modified_time.column.label: "lmt",
drill.exec.storage.implicit.project_metadata.column.label: "$project_metadata$",
- drill.exec.implicit_casts.joins.enabled: false,
+ drill.exec.storage.implicit.row_group_index.column.label: "rgi",
+ drill.exec.storage.implicit.row_group_length.column.label: "rgl",
+ drill.exec.storage.implicit.row_group_start.column.label: "rgs",
+ drill.exec.storage.implicit.suffix.column.label: "suffix",
drill.exec.testing.controls: "{}",
- drill.exec.memory.operator.output_batch_size : 16777216, # 16 MB
- drill.exec.memory.operator.output_batch_size_avail_mem_factor : 0.1,
exec.bulk_load_table_list.bulk_size: 1000,
+ exec.enable_aliases: true,
exec.enable_bulk_load_table_list: false,
+ exec.enable_dynamic_fc: false,
exec.enable_union_type: false,
- exec.except_add_agg_below: false,
exec.errors.verbose: false,
- drill.exec.http.rest.errors.verbose: false,
- exec.hashjoin.mem_limit: 0,
- exec.hashjoin.hash_table_calc_type: "LEAN",
- exec.hashjoin.safety_factor: 1.0,
- exec.hashjoin.fragmentation_factor: 1.33,
- exec.hashjoin.hash_double_factor: 2.0,
- exec.hashjoin.num_partitions: 32,
- exec.hashjoin.num_rows_in_batch: 1024,
- exec.hashjoin.max_batches_in_memory: 0,
- exec.hashjoin.enable.runtime_filter: false,
- exec.hashjoin.bloom_filter.fpp: 0.75,
- exec.hashjoin.bloom_filter.max.size: 33554432, #32 MB
- exec.hashjoin.runtime_filter.waiting.enable: true,
- exec.hashjoin.runtime_filter.max.waiting.time: 300, #400 ms
+ exec.except_add_agg_below: false,
+
+ exec.hashagg.max_batches_in_memory: 65536,
exec.hashagg.mem_limit: 0,
exec.hashagg.min_batches_per_partition: 2,
exec.hashagg.num_partitions: 32,
exec.hashagg.num_rows_in_batch: 128,
- exec.hashagg.max_batches_in_memory: 65536,
exec.hashagg.use_memory_prediction: true,
+
+ exec.hashjoin.bloom_filter.fpp: 0.75,
+ exec.hashjoin.bloom_filter.max.size: 33554432, #32 MB
+ exec.hashjoin.enable.runtime_filter: false,
+ exec.hashjoin.fragmentation_factor: 1.33,
+ exec.hashjoin.hash_double_factor: 2.0,
+ exec.hashjoin.hash_table_calc_type: "LEAN",
+ exec.hashjoin.max_batches_in_memory: 0,
+ exec.hashjoin.mem_limit: 0,
+ exec.hashjoin.num_partitions: 32,
+ exec.hashjoin.num_rows_in_batch: 1024,
+ exec.hashjoin.runtime_filter.max.waiting.time: 300, #400 ms
+ exec.hashjoin.runtime_filter.waiting.enable: true,
+ exec.hashjoin.safety_factor: 1.0,
+
exec.impersonation.inbound_policies: "[]",
- exec.java.compiler.exp_in_method_size: 50,
- exec.java_compiler: "DEFAULT",
+
exec.java_compiler_debug: true,
+ exec.java_compiler: "DEFAULT",
+ exec.java.compiler.exp_in_method_size: 50,
exec.java_compiler_janino_maxsize: 262144,
+
exec.max_hash_table_size: 1073741824,
exec.min_hash_table_size: 65536,
exec.partition.mem_throttle: 0,
exec.persistent_table.umask: "002",
- exec.query.progress.update: true,
+
+ exec.query_profile.alter_session.skip: true,
exec.query_profile.debug_mode: false,
exec.query_profile.save: true,
- exec.query_profile.alter_session.skip: true,
- exec.enable_dynamic_fc: false,
- exec.enable_aliases: true,
- exec.queue.enable: false,
+
+ exec.query.progress.update: true,
+
# Default queue values for an 8 GB direct memory default
# Drill install. Users are expected to adjust these based
# on load and available resources.
+ exec.queue.enable: false,
exec.queue.large: 2,
+ exec.queue.memory_ratio: 10.0,
+ exec.queue.memory_reserve_ratio: 0.2,
exec.queue.small: 4,
exec.queue.threshold: 30000000,
exec.queue.timeout_millis: 300000,
- exec.queue.memory_ratio: 10.0,
- exec.queue.memory_reserve_ratio: 0.2,
+
# Deprecated
exec.sort.disable_managed : false,
# Deprecated
@@ -616,11 +630,9 @@
exec.storage.skip_runtime_rowgroup_pruning: false,
exec.udf.enable_dynamic_support: true,
exec.udf.use_dynamic: true,
- drill.exec.stats.logging.batch_size: false,
- drill.exec.stats.logging.fine_grained.batch_size: false,
- drill.exec.stats.logging.enabled_operators: all,
new_view_default_permissions: 700,
org.apache.drill.exec.compile.ClassTransformer.scalar_replacement: "try",
+
planner.add_producer_consumer: false,
planner.affinity_factor: 1.2,
planner.broadcast_factor: 1.0,
@@ -631,26 +643,24 @@
planner.enable_constant_folding: true,
planner.enable_decimal_data_type: true,
planner.enable_demux_exchange: false,
- planner.enable_hash_single_key: true,
+ planner.enable_file_listing_limit0_optimization: false,
planner.enable_hashagg: true,
- planner.enable_hashjoin: true,
- planner.enable_semijoin: true,
planner.enable_hashjoin_swap: true,
+ planner.enable_hashjoin: true,
+ planner.enable_hash_single_key: true,
planner.enable_hep_opt: true,
planner.enable_hep_partition_pruning: true,
- planner.enable_rowkeyjoin_conversion: true,
- planner.rowkeyjoin_conversion_using_hashjoin: false,
- planner.rowkeyjoin_conversion_selectivity_threshold: 0.01,
planner.enable_join_optimization: true,
- planner.enable_limit0_optimization: true,
planner.enable_limit0_on_scan: true,
- planner.enable_file_listing_limit0_optimization: false,
+ planner.enable_limit0_optimization: true,
planner.enable_mergejoin: true,
planner.enable_multiphase_agg: true,
planner.enable_mux_exchange: true,
- planner.enable_ordered_mux_exchange: true,
planner.enable_nestedloopjoin: true,
planner.enable_nljoin_for_scalar_only: true,
+ planner.enable_ordered_mux_exchange: true,
+ planner.enable_rowkeyjoin_conversion: true,
+ planner.enable_semijoin: true,
planner.enable_streamagg: true,
planner.enable_topn: true,
planner.enable_type_inference: true,
@@ -667,132 +677,142 @@
planner.memory.enable_memory_estimation: false,
planner.memory.hash_agg_table_factor: 1.1d,
planner.memory.hash_join_table_factor: 1.1d,
+ planner.memory_limit: 268435456,
planner.memory.max_query_memory_per_node: 2147483648, # 2 GB
- planner.memory.percent_per_query: 0.05, # 5%
planner.memory.min_memory_per_buffered_op: 41943040, # 40 MB
planner.memory.non_blocking_operators_memory: 64,
- planner.memory_limit: 268435456,
+ planner.memory.percent_per_query: 0.05, # 5%
planner.nestedloopjoin_factor: 100.0,
planner.parser.quoting_identifiers: "`",
planner.partitioner_sender_max_threads: 8,
planner.partitioner_sender_set_threads: -1,
planner.partitioner_sender_threads_factor: 2,
planner.producer_consumer_queue_size: 10,
+ planner.rowkeyjoin_conversion_selectivity_threshold: 0.01,
+ planner.rowkeyjoin_conversion_using_hashjoin: false,
planner.slice_target: 100000,
- planner.statistics.use: false,
planner.statistics.multicol_ndv_adjustment_factor: 1.0,
+ planner.statistics.use: false,
planner.store.parquet.rowgroup.filter.pushdown.enabled: true,
planner.store.parquet.rowgroup.filter.pushdown.threshold: 10000,
# Max per node should always be configured as zero and
# it is dynamically computed based on cpu_load_average
planner.width.max_per_node: 0,
planner.width.max_per_query: 1000,
+
prepare.statement.create_timeout_ms: 30000,
security.admin.user_groups: "%drill_process_user_groups%",
security.admin.users: "%drill_process_user%",
+ storage.list_files_recursively: false,
+ storage.plugin_auto_disable: false,
+ storage.plugin_retry_attempt_delay: 2000,
+ storage.plugin_retry_attempts: 1,
store.format: "parquet",
- store.hive.optimize_scan_with_native_readers: false,
- store.hive.parquet.optimize_scan_with_native_reader: false,
# Properties values should NOT be set in double-quotes or any other quotes.
# Property name and value should be separated by =.
# Properties should be separated by new line (\n).
store.hive.conf.properties: "",
- store.json.enable_v2_reader: true,
+ store.hive.optimize_scan_with_native_readers: false,
+ store.hive.parquet.optimize_scan_with_native_reader: false,
store.json.all_text_mode: false,
- store.json.writer.allow_nan_inf: true,
- store.json.reader.allow_nan_inf: true,
- store.json.reader.allow_escape_any_char: false,
+ store.json.enable_v2_reader: true,
store.json.extended_types: false,
- store.json.read_numbers_as_double: false,
+ store.json.reader.allow_escape_any_char: false,
+ store.json.reader.allow_nan_inf: true,
store.json.reader.print_skipped_invalid_record_number: false,
store.json.reader.skip_invalid_records: false,
+ store.json.read_numbers_as_double: false,
+ store.json.writer.allow_nan_inf: true,
store.json.writer.skip_null_fields: true,
store.json.writer.uglify: false,
+ store.kafka.all_text_mode: false,
+ store.kafka.poll.timeout: 200,
+ store.kafka.reader.allow_escape_any_char: false,
+ store.kafka.reader.allow_nan_inf: true,
+ store.kafka.reader.skip_invalid_records: false,
+ store.kafka.read_numbers_as_double: false,
+ store.kafka.record.reader: "org.apache.drill.exec.store.kafka.decoders.JsonMessageReader",
store.mongo.all_text_mode: false,
store.mongo.bson.record.reader: true,
store.mongo.read_numbers_as_double: false,
store.parquet.block-size: 536870912,
+
+ store.parquet.complex.batch.num_records: 4000,
store.parquet.compression: "none",
store.parquet.dictionary.page-size: 1048576,
store.parquet.enable_dictionary_encoding: false,
+ # Using common operators batch configuration unless the Parquet specific
+ # configuration is used
+ store.parquet.flat.batch.memory_size: 0,
+ store.parquet.flat.batch.num_records: 32767,
+ store.parquet.flat.reader.bulk: true,
store.parquet.page-size: 1048576,
store.parquet.reader.columnreader.async: false,
+ store.parquet.reader.enable_map_support: true,
store.parquet.reader.int96_as_timestamp: false,
- store.parquet.reader.strings_signed_min_max: "",
store.parquet.reader.pagereader.async: true,
store.parquet.reader.pagereader.bufferedread: true,
store.parquet.reader.pagereader.buffersize: 1048576,
store.parquet.reader.pagereader.enforceTotalSize: false,
store.parquet.reader.pagereader.queuesize: 2,
store.parquet.reader.pagereader.usefadvise: false,
- store.parquet.reader.enable_map_support: true,
+ store.parquet.reader.strings_signed_min_max: "",
store.parquet.use_new_reader: false,
store.parquet.vector_fill_check_threshold: 10,
store.parquet.vector_fill_threshold: 85,
- store.parquet.writer.use_primitive_types_for_decimals: true,
- store.parquet.writer.logical_type_for_decimals: "fixed_len_byte_array",
- store.parquet.writer.use_single_fs_block: false,
store.parquet.writer.format_version: "v1",
- store.parquet.flat.reader.bulk: true,
- store.parquet.flat.batch.num_records: 32767,
- store.parquet.complex.batch.num_records: 4000,
- # Using common operators batch configuration unless the Parquet specific
- # configuration is used
- store.parquet.flat.batch.memory_size: 0,
- store.table.use_schema_file: false,
+ store.parquet.writer.logical_type_for_decimals: "fixed_len_byte_array",
+ store.parquet.writer.use_primitive_types_for_decimals: true,
+ store.parquet.writer.use_single_fs_block: false,
+
store.partition.hash_distribute: false,
+ store.table.use_schema_file: false,
store.text.estimated_row_size_bytes: 100.0,
store.text.writer.add_header: true,
store.text.writer.force_quotes: false,
- store.kafka.all_text_mode: false,
- store.kafka.read_numbers_as_double: false,
- store.kafka.record.reader: "org.apache.drill.exec.store.kafka.decoders.JsonMessageReader",
- store.kafka.poll.timeout: 200,
- store.kafka.reader.skip_invalid_records: false,
- store.kafka.reader.allow_nan_inf: true,
- store.kafka.reader.allow_escape_any_char: false,
- web.logs.max_lines: 10000,
- web.display_format.timestamp: "",
web.display_format.date: "",
web.display_format.time: "",
+ web.display_format.timestamp: "",
+ web.logs.max_lines: 10000,
window.enable: true,
- storage.list_files_recursively: false,
- storage.plugin_retry_attempts: 1,
- storage.plugin_retry_attempt_delay: 2000,
- storage.plugin_auto_disable: false,
+
# ============ index plan related options ==============
- planner.use_simple_optimizer: false,
+ planner.disable_full_table_scan: false,
planner.enable_index_planning: true,
planner.enable_statistics: true,
- planner.disable_full_table_scan: false,
- planner.index.max_chosen_indexes_per_table: 5,
- planner.index.force_sort_noncovering: false,
- planner.index.use_hashjoin_noncovering: false,
planner.index.covering_selectivity_threshold: 1.0,
+ planner.index.force_sort_noncovering: false,
+ planner.index.max_chosen_indexes_per_table: 5,
+ planner.index.max_indexes_to_intersect: 5,
planner.index.noncovering_selectivity_threshold: 0.025,
+ planner.index.prefer_intersect_plans: false,
planner.index.rowkeyjoin_cost_factor: 0.1,
planner.index.statistics_rowcount_scaling_factor: 0.1,
- planner.index.prefer_intersect_plans: false,
- planner.index.max_indexes_to_intersect: 5,
- exec.query.rowkeyjoin_batchsize: 128,
- exec.query.return_result_set_for_ddl: true,
+ planner.index.use_hashjoin_noncovering: false,
+ planner.use_simple_optimizer: false,
+
exec.query.max_rows: 0,
- exec.return_result_set_for_ddl: true,
- exec.statistics.ndv_accuracy: 20,
- exec.statistics.ndv_extrapolation_bf_elements: 1000000,
- exec.statistics.ndv_extrapolation_bf_fpprobability: 10,
- exec.statistics.deterministic_sampling: false,
exec.query.return_result_set_for_ddl: true,
+ exec.query.return_result_set_for_ddl: true,
+ exec.query.rowkeyjoin_batchsize: 128,
+ exec.return_result_set_for_ddl: true,
+
# ========= rm related options ===========
exec.rm.queryTags: "",
exec.rm.queues.wait_for_preferred_nodes: true,
+
+ exec.statistics.deterministic_sampling: false,
+ exec.statistics.ndv_accuracy: 20,
+ exec.statistics.ndv_extrapolation_bf_elements: 1000000,
+ exec.statistics.ndv_extrapolation_bf_fpprobability: 10,
exec.statistics.tdigest_compression: 100,
+
# ========= Metastore related options ===========
metastore.enabled: false,
- metastore.retrieval.retry_attempts: 5
+ metastore.metadata.ctas.auto-collect: "NONE",
+ metastore.metadata.fallback_to_file_metadata: true
metastore.metadata.store.depth_level: "ALL",
metastore.metadata.use_schema: true,
metastore.metadata.use_statistics: true,
- metastore.metadata.ctas.auto-collect: "NONE",
- metastore.metadata.fallback_to_file_metadata: true
+ metastore.retrieval.retry_attempts: 5
}
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index 79e799a..573853b 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -882,7 +882,7 @@
<profile>
<id>hadoop-2</id>
<properties>
- <jdbc-all-jar.maxsize>54400000</jdbc-all-jar.maxsize>
+ <jdbc-all-jar.maxsize>54500000</jdbc-all-jar.maxsize>
</properties>
</profile>
</profiles>
diff --git a/pom.xml b/pom.xml
index 83fb666..70cb66b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,7 +57,7 @@
<avatica.version>1.23.0</avatica.version>
<janino.version>3.1.11</janino.version>
<sqlline.version>1.12.0</sqlline.version>
- <jackson.version>2.14.3</jackson.version>
+ <jackson.version>2.16.1</jackson.version>
<zookeeper.version>3.5.10</zookeeper.version>
<kerby.version>1.0.0</kerby.version>
<findbugs.version>3.0.5</findbugs.version>
