| // Licensed to the Apache Software Foundation (ASF) under one or more |
| // contributor license agreements. See the NOTICE file distributed with |
| // this work for additional information regarding copyright ownership. |
| // The ASF licenses this file to You under the Apache License, Version 2.0 |
| // (the "License"); you may not use this file except in compliance with |
| // the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| // This file tells Drill to consider this module when class path scanning. |
| // This file can also include any supplementary configuration information. |
| // This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. |
| |
| drill { |
| classpath.scanning { |
| base.classes: ${?drill.classpath.scanning.base.classes} [ |
| org.apache.drill.exec.expr.DrillFunc, |
| org.apache.drill.exec.expr.fn.PluggableFunctionRegistry, |
| org.apache.drill.exec.physical.base.PhysicalOperator, |
| org.apache.drill.exec.physical.impl.BatchCreator, |
| org.apache.drill.exec.physical.impl.RootCreator, |
| org.apache.drill.exec.rpc.user.security.UserAuthenticator, |
| org.apache.drill.exec.rpc.security.AuthenticatorFactory, |
| org.apache.drill.exec.server.rest.auth.DrillHttpConstraintSecurityHandler, |
| org.apache.drill.exec.store.dfs.FormatPlugin, |
| org.apache.drill.exec.store.StoragePlugin |
| ], |
| |
| annotations += org.apache.drill.exec.expr.annotations.FunctionTemplate |
| |
| packages: ${?drill.classpath.scanning.packages} [ |
| org.apache.drill.exec.expr, |
| org.apache.drill.exec.physical, |
| org.apache.drill.exec.store, |
| org.apache.drill.exec.rpc.user.security, |
| org.apache.drill.exec.rpc.security, |
| org.apache.drill.exec.server.rest.auth |
| ] |
| } |
| } |
| |
| drill.client: { |
| supports-complex-types: true |
| } |
| |
| // Location Drill uses for temporary files, such as downloaded dynamic UDFs jars. |
| // By default ${DRILL_TMP_DIR} is used if set or ${drill.tmp-dir} if it's been overridden. |
| drill.tmp-dir: "/tmp" |
| drill.tmp-dir: ${?DRILL_TMP_DIR} |
| drill.exec: { |
| cluster-id: "drillbits1", |
| rpc: { |
| user: { |
| timeout: 30, |
| server: { |
| port: 31010 |
| threads: 1, |
| memory: { |
| reservation: 0, |
| maximum: 9223372036854775807 |
| } |
| } |
| client: { |
| threads: 1 |
| } |
| }, |
| bit: { |
| timeout: 300, |
| server: { |
| port: 31011, |
| retry:{ |
| count: 7200, |
| delay: 500 |
| }, |
| threads: 10 |
| memory: { |
| control: { |
| reservation: 0, |
| maximum: 9223372036854775807 |
| }, |
| data: { |
| reservation: 0, |
| maximum: 9223372036854775807 |
| } |
| } |
| } |
| }, |
| use.ip: false |
| }, |
| optimizer: { |
| implementation: "org.apache.drill.exec.opt.IdentityOptimizer" |
| }, |
| storage: { |
| registry: "org.apache.drill.exec.store.StoragePluginRegistryImpl", |
| file: { |
| text: { |
| buffer.size: 262144, |
| batch.size: 4000 |
| } |
| } |
| }, |
| zk: { |
| connect: "localhost:2181", |
| root: "drill", |
| refresh: 500, |
| timeout: 5000, |
| retry: { |
| count: 7200, |
| delay: 500 |
| } |
| }, |
| http: { |
| enabled: true, |
| ssl_enabled: false, |
| porthunt: false, |
| port: 8047, |
| jetty : { |
| server : { |
| acceptors : 1, |
| selectors : 2 |
| } |
| } |
| max_profiles: 100, |
| session_max_idle_secs: 3600, # Default value 1hr |
| cors: { |
| enabled: false, |
| allowedOrigins: ["null"], |
| allowedMethods: ["GET", "POST", "HEAD", "OPTIONS"], |
| allowedHeaders: ["X-Requested-With", "Content-Type", "Accept", "Origin"], |
| credentials: true |
| }, |
| session: { |
| memory: { |
| reservation: 0, |
| maximum: 9223372036854775807 |
| } |
| } |
| }, |
| //setting javax variables for ssl configurations is being deprecated. |
| ssl: { |
| keyStoreType = ${?javax.net.ssl.keyStoreType}, |
| keyStorePath = ${?javax.net.ssl.keyStore}, |
| keyStorePassword = ${?javax.net.ssl.keyStorePassword}, |
| trustStoreType = ${?javax.net.ssl.trustStoreType}, |
| trustStorePath = ${?javax.net.ssl.trustStore}, |
| trustStorePassword = ${?javax.net.ssl.trustStorePassword} |
| # default key password to keystore password |
| keyPassword = ${?javax.net.ssl.keyStorePassword}, |
| protocol: "TLSv1.2", |
| # if true, then Drill will read SSL parameters from the |
| # Hadoop configuration files. |
| useHadoopConfig : true, |
| #Drill can use either the JDK implementation or the OpenSSL implementation. |
| provider: "JDK" |
| }, |
| network: { |
| start: 35000 |
| }, |
| work: { |
| affinity.factor: 1.2 |
| }, |
| sys.store.provider: { |
| class: "org.apache.drill.exec.store.sys.store.provider.ZookeeperPersistentStoreProvider", |
| local: { |
| path: "/tmp/drill", |
| write: true |
| } |
| }, |
| profiles.store: { |
| inmemory: false, |
| capacity: 1000 |
| }, |
| impersonation: { |
| enabled: false, |
| max_chained_user_hops: 3 |
| }, |
| security.user.auth: { |
| enabled: false |
| }, |
| security.bit.auth: { |
| enabled: false |
| use_login_principal: false |
| } |
| security.user.encryption.sasl: { |
| enabled: false, |
| max_wrapped_size: 65536 |
| } |
| security.bit.encryption.sasl: { |
| enabled: false, |
| max_wrapped_size: 65536 |
| } |
| security.user.encryption.ssl: { |
| enabled: false |
| } |
| trace: { |
| directory: "/tmp/drill-trace", |
| filesystem: "file:///" |
| }, |
| tmp: { |
| directories: ["/tmp/drill"], |
| filesystem: "drill-local:///" |
| }, |
| buffer:{ |
| size: "6", |
| spooling: { |
| delete: true, |
| size: 100000000 |
| } |
| }, |
| compile: { |
| codegen.debug.topn: false, |
| compiler: "DEFAULT", |
| debug: true, |
| janino_maxsize: 262144, |
| cache_max_size: 1000, |
| // Where to save the generated source. See ClassBuilder |
| code_dir: "/tmp/drill/codegen" |
| // Disable code cache. Only for testing. |
| disable_cache: false, |
| // Use plain Java compilation where available |
| prefer_plain_java: false |
| }, |
| debug: { |
| // If true, inserts the iterator validator atop each operator. |
| // Primrily used for testing. |
| validate_iterators: false, |
| // If iterator validation is enabled, also validates the vectors |
| // in each batch. Primarily used for testing. To enable from |
| // the command line: |
| // java ... -ea -Ddrill.exec.debug.validate_vectors=true ... |
| validate_vectors: false |
| }, |
| spill: { |
| // *** Options common to all the operators that may spill |
| // File system to use. Local file system by default. |
| fs: "file:///", |
| // List of directories to use. Directories are created |
| // if they do not exist. |
| directories: [ "/tmp/drill/spill" ] |
| }, |
| hashagg: { |
| spill: { |
| // -- The 2 options below can be used to override the common ones |
| // -- (common to all spilling operators) |
| // File system to use. Local file system by default. |
| fs: ${drill.exec.spill.fs}, |
| // List of directories to use. Directories are created |
| // if they do not exist. |
| directories: ${drill.exec.spill.directories}, |
| } |
| }, |
| sort: { |
| purge.threshold : 10, |
| external: { |
| // Limit on the number of batches buffered in memory. |
| // Primarily for testing. |
| // 0 = unlimited |
| batch_limit: 0, |
| // Limit on the amount of memory used for xsort. Overrides the |
| // value provided by Foreman. Primarily for testing. |
| // 0 = unlimited, Supports HOCON memory suffixes. |
| mem_limit: 0, |
| // Limit on the number of batches that can be merged in |
| // a single pass. Limits the number of open file handles. |
| // 0 = unlimited |
| merge_limit: 128, |
| spill: { |
| // Deprecated for managed xsort; used only by legacy xsort |
| group.size: 40000, |
| // Deprecated for managed xsort; used only by legacy xsort |
| threshold: 40000, |
| // -- The two options below can be used to override the options common |
| // -- for all spilling operators (see "spill" above). |
| // -- This is done for backward compatibility; in the future they |
| // -- would be deprecated (you should be using only the common ones) |
| // File system to use. Local file system by default. |
| fs: ${drill.exec.spill.fs}, |
| // List of directories to use. Directories are created |
| // if they do not exist. |
| directories: ${drill.exec.spill.directories}, |
| // Size of the batches written to, and read from, the spill files. |
| // Determines the ratio of memory to input data size for a single- |
| // generation sort. Smaller values are better, but too small |
| // incurs per-batch overhead. |
| spill_batch_size = 1M, |
| // Preferred file size for "first-generation" spill files. |
| // Set large enough to get long, continuous writes, but not so |
| // large as to overwhelm a temp directory. |
| // Supports HOCON memory suffixes. |
| file_size: 256M |
| } |
| } |
| }, |
| queue : { |
| // Settings for the local query queue available for embedded drillbits. |
| embedded : { |
| // Enable the local query queue |
| enable: false, |
| // Maximum number of queries that can run simultaneously. |
| // All others queue. |
| size: 2, |
| // Maximum wait time in the queue before the query times out and |
| // fails. |
| timeout_ms: 5000 // 5 seconds |
| } |
| } |
| memory: { |
| operator: { |
| max: 20000000000, |
| initial: 10000000 |
| }, |
| fragment: { |
| max: 20000000000, |
| initial: 20000000 |
| } |
| }, |
| debug: { |
| return_error_for_failure_in_cancelled_fragments: false |
| }, |
| scan: { |
| threadpool_size: 8, |
| decode_threadpool_size: 1 |
| }, |
| udf: { |
| retry-attempts: 5, |
| // Disables (parts of) the dynamic UDF functionality. |
| // Primarily for testing. |
| disable_dynamic: false, |
| directory: { |
| // Base directory for remote and local udf directories, unique among clusters. |
| base: ${drill.exec.zk.root}"/udf", |
| |
| // Path to local udf directory, always created on local file system. |
| // Root for these directory is generated at runtime unless Drill temporary directory is set. |
| local: ${drill.exec.udf.directory.base}"/udf/local", |
| |
| // Set this property if custom file system should be used to create remote directories, ex: fs: "file:///". |
| // fs: "", |
| // Set this property if custom absolute root should be used for remote directories, ex: root: "/app/drill". |
| // root: "", |
| |
| // Relative path to all remote udf directories. |
| // Directories are created under default file system taken from Hadoop configuration |
| // unless ${drill.exec.udf.directory.fs} is set. |
| // User home directory is used as root unless ${drill.exec.udf.directory.root} is set. |
| staging: ${drill.exec.udf.directory.base}"/staging", |
| registry: ${drill.exec.udf.directory.base}"/registry", |
| tmp: ${drill.exec.udf.directory.base}"/tmp" |
| } |
| }, |
| # Temporary table can be created ONLY in default temporary workspace. |
| # Full workspace name should be indicated (including schema and workspace separated by dot). |
| # Workspace MUST be file-based and writable. Workspace name is case-sensitive. |
| default_temporary_workspace: "dfs.tmp" |
| |
| // Resource management |
| rm : { |
| // Memory per node normally comes from the direct memory alloated on the JVM |
| // command line. This parameter, if other than 0, further limits the amount. |
| // Primarily for testing. |
| memory_per_node: 0, |
| // The number of available CPUs normally comes directly from the system itself. |
| // This parameter, if other than 0, further limits the number of CPUs will |
| // will consider when planning. Note that, sadly, this parameter does not |
| // limit *actual* CPU usage; only the amount of CPU assumed to exist when |
| // planning and managing queries. Primarily for testing. |
| cpus_per_node: 0, |
| } |
| # Grace period is the amount of time where the drillbit accepts work after |
| # the shutdown request is triggered. The primary use of grace period is to |
| # avoid the race conditions caused by zookeeper delay in updating the state |
| # information of the drillbit that is shutting down. So, it is advisable |
| # to have a grace period that is atleast twice the amount of zookeeper |
| # refresh time. |
| grace_period_ms : 0, |
| //port hunting for drillbits. Enabled only for testing purposes. |
| port_hunt : false |
| } |
| |
| drill.jdbc: { |
| batch_queue_throttling_threshold: 100 |
| } |
| |
| # The following are defaults for system and session options. |
| # Provided here for easy customization in Drill distributions |
| # like the drill-distrib.conf file. |
| # An addition of new system option should go through following procedure. |
| # Before adding a system option a validator should be added for that |
| # option and then the option should be configured below under the name |
| # space drill.exec.options. When the system options are being loaded |
| # options config options are read to find the default value for a give option |
| # if the option is not set using ALTER SYSTEM or ALTER SESSION.But |
| # if the option is not configured in the conf file under the name space |
| # "drill.exec.options", option missing exception will be thrown. |
| # Users are not supposed to set these options in the drill-override.conf file. |
| # Users should use ALTER SYSTEM and ALTER SESSION to set the options. |
| |
| drill.exec.options: { |
| bootstrap-storage-plugins.json: .sys.drill, |
| debug.validate_iterators: false, |
| debug.validate_vectors: false, |
| drill.exec.functions.cast_empty_string_to_null: false, |
| drill.exec.rpc.fragrunner.timeout: 10000, |
| # Setting to control if HashAgg should fallback to older behavior of consuming |
| # unbounded memory. In case of 2 phase Agg when available memory is not enough |
| # to start at least 2 partitions then HashAgg fallbacks to this case. It can be |
| # enabled by setting this flag to true. By default it's set to false such that |
| # query will fail if there is not enough memory |
| drill.exec.hashagg.fallback.enabled: false, |
| drill.exec.storage.file.partition.column.label: "dir", |
| drill.exec.storage.implicit.filename.column.label: "filename", |
| drill.exec.storage.implicit.filepath.column.label: "filepath", |
| drill.exec.storage.implicit.fqn.column.label: "fqn", |
| drill.exec.storage.implicit.suffix.column.label: "suffix", |
| drill.exec.testing.controls: "{}", |
| drill.exec.memory.operator.output_batch_size : 16777216, # 16 MB |
| exec.bulk_load_table_list.bulk_size: 1000, |
| exec.compile.scalar_replacement: false, |
| exec.enable_bulk_load_table_list: false, |
| exec.enable_union_type: false, |
| exec.errors.verbose: false, |
| exec.hashagg.mem_limit: 0, |
| exec.hashagg.min_batches_per_partition: 2, |
| exec.hashagg.num_partitions: 32, |
| exec.hashagg.use_memory_prediction: true, |
| exec.impersonation.inbound_policies: "[]", |
| exec.java.compiler.exp_in_method_size: 50, |
| exec.java_compiler: "DEFAULT", |
| exec.java_compiler_debug: true, |
| exec.java_compiler_janino_maxsize: 262144, |
| exec.max_hash_table_size: 1073741824, |
| exec.min_hash_table_size: 65536, |
| exec.persistent_table.umask: "002", |
| exec.query.progress.update: true, |
| exec.query_profile.debug_mode: false, |
| exec.query_profile.save: true, |
| exec.queue.enable: false, |
| # Default queue values for an 8 GB direct memory default |
| # Drill install. Users are expected to adjust these based |
| # on load and available resources. |
| exec.queue.large: 2, |
| exec.queue.small: 4, |
| exec.queue.threshold: 30000000, |
| exec.queue.timeout_millis: 300000, |
| exec.queue.memory_ratio: 10.0, |
| exec.queue.memory_reserve_ratio: 0.2, |
| exec.sort.disable_managed : false, |
| exec.storage.enable_new_text_reader: true, |
| exec.udf.enable_dynamic_support: true, |
| exec.udf.use_dynamic: true, |
| new_view_default_permissions: 700, |
| org.apache.drill.exec.compile.ClassTransformer.scalar_replacement: "try", |
| planner.add_producer_consumer: false, |
| planner.affinity_factor: 1.2, |
| planner.broadcast_factor: 1.0, |
| planner.broadcast_threshold: 10000000, |
| planner.cpu_load_average: 0.70, |
| planner.disable_exchanges: false, |
| planner.enable_broadcast_join: true, |
| planner.enable_constant_folding: true, |
| planner.enable_decimal_data_type: false, |
| planner.enable_demux_exchange: false, |
| planner.enable_hash_single_key: true, |
| planner.enable_hashagg: true, |
| planner.enable_hashjoin: true, |
| planner.enable_hashjoin_swap: true, |
| planner.enable_hep_opt: true, |
| planner.enable_hep_partition_pruning: true, |
| planner.enable_join_optimization: true, |
| planner.enable_limit0_optimization: false, |
| planner.enable_mergejoin: true, |
| planner.enable_multiphase_agg: true, |
| planner.enable_mux_exchange: true, |
| planner.enable_ordered_mux_exchange: true, |
| planner.enable_nestedloopjoin: true, |
| planner.enable_nljoin_for_scalar_only: true, |
| planner.enable_streamagg: true, |
| planner.enable_type_inference: true, |
| planner.enable_unionall_distribute: false, |
| planner.filter.max_selectivity_estimate_factor: 1.0, |
| planner.filter.min_selectivity_estimate_factor: 0.0, |
| planner.force_2phase_aggr: false, |
| planner.identifier_max_length: 1024, |
| planner.in_subquery_threshold: 20, |
| planner.join.hash_join_swap_margin_factor: 10, |
| planner.join.row_count_estimate_factor: 1.0, |
| planner.memory.average_field_width: 8, |
| planner.memory.enable_memory_estimation: false, |
| planner.memory.hash_agg_table_factor: 1.1d, |
| planner.memory.hash_join_table_factor: 1.1d, |
| planner.memory.max_query_memory_per_node: 2147483648, # 2 GB |
| planner.memory.percent_per_query: 0.05, # 5% |
| planner.memory.min_memory_per_buffered_op: 41943040, # 40 MB |
| planner.memory.non_blocking_operators_memory: 64, |
| planner.memory_limit: 268435456, |
| planner.nestedloopjoin_factor: 100.0, |
| planner.parser.quoting_identifiers: "`", |
| planner.partitioner_sender_max_threads: 8, |
| planner.partitioner_sender_set_threads: -1, |
| planner.partitioner_sender_threads_factor: 2, |
| planner.producer_consumer_queue_size: 10, |
| planner.slice_target: 100000, |
| planner.store.parquet.rowgroup.filter.pushdown.enabled: true, |
| planner.store.parquet.rowgroup.filter.pushdown.threshold: 10000, |
| # Max per node should always be configured as zero and |
| # it is dynamically computed based on cpu_load_average |
| planner.width.max_per_node: 0, |
| planner.width.max_per_query: 1000, |
| prepare.statement.create_timeout_ms: 10000, |
| security.admin.user_groups: "%drill_process_user_groups%", |
| security.admin.users: "%drill_process_user%", |
| store.format: "parquet", |
| store.hive.optimize_scan_with_native_readers: false, |
| store.json.all_text_mode: false, |
| store.json.writer.allow_nan_inf: true, |
| store.json.reader.allow_nan_inf: true, |
| store.json.extended_types: false, |
| store.json.read_numbers_as_double: false, |
| store.json.reader.print_skipped_invalid_record_number: false, |
| store.json.reader.skip_invalid_records: false, |
| store.json.writer.skip_null_fields: true, |
| store.json.writer.uglify: false, |
| store.mongo.all_text_mode: false, |
| store.mongo.bson.record.reader: true, |
| store.mongo.read_numbers_as_double: false, |
| store.parquet.block-size: 536870912, |
| store.parquet.compression: "none", |
| store.parquet.dictionary.page-size: 1048576, |
| store.parquet.enable_dictionary_encoding: false, |
| store.parquet.page-size: 1048576, |
| store.parquet.reader.columnreader.async: false, |
| store.parquet.reader.int96_as_timestamp: false, |
| store.parquet.reader.pagereader.async: true, |
| store.parquet.reader.pagereader.bufferedread: true, |
| store.parquet.reader.pagereader.buffersize: 1048576, |
| store.parquet.reader.pagereader.enforceTotalSize: false, |
| store.parquet.reader.pagereader.queuesize: 2, |
| store.parquet.reader.pagereader.usefadvise: false, |
| store.parquet.use_new_reader: false, |
| store.parquet.vector_fill_check_threshold: 10, |
| store.parquet.vector_fill_threshold: 85, |
| store.parquet.writer.use_single_fs_block: false, |
| store.partition.hash_distribute: false, |
| store.text.estimated_row_size_bytes: 100.0, |
| store.kafka.all_text_mode: false, |
| store.kafka.read_numbers_as_double: false, |
| store.kafka.record.reader: "org.apache.drill.exec.store.kafka.decoders.JsonMessageReader", |
| store.kafka.poll.timeout: 200, |
| web.logs.max_lines: 10000, |
| window.enable: true, |
| } |
