distribution/src/main/resources/drill-on-yarn-example.conf

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Configuration for the Drill-on-YARN feature. Provides information needed to
# launch the Drill Application Master under YARN, and to configure the
# Drill cluster.
#
# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md
# for more information.
#
# To create your own configuration file, copy only those lines that you want
# to change. All values have reasonable defaults.

drill.yarn: {

  # Name displayed in the YARN management UI for your Drill cluster.

  app-name: "Drill-on-YARN"

  dfs: {
    # Connection to the distributed file system. Defaults to work with
    # a single-node Drill on the local machine.
    # Omit this if you want to get the configuration either from the
    # Hadoop config (set with config-dir above) or from the
    # $DRILL_HOME/core-site.xml.

    connection: "hdfs://localhost/"

    # The distributed file system (DFS such as HDFS) directory in which to store the Drill
    # archive file. Change this if you launch multiple Drill clusters to give
    # each cluster it's own upload location.

    app-dir: "/users/drill"
  }

  yarn: {

    # Specify the queue you want to use when launching Drill-on-YARN.

    queue: "default"
  }

  drill-install: {

    # The archive (in .tar.gz format) that contains your Drill software, your custom jar files,
    # and your Drill configuration. This is the path the file on your client machine.
    # The base name is used when uploading the file to DFS.

    client-path: "/etc/drill/drill.tar.gz"

    # Set this if the name of the directory inside your archive file differs from
    # the name of the file. That is, by default, my-drill.tar.gz should contain
    # my-drill as its directory name.

    # dir-name: "drill"

    # Note that there are no parameters for the site directory. You specify the
    # site directory location on client launch; Drill-on-YARN automatically
    # creates an archive of that directory.

    # Set the Java java.library.path option to files that pre-exist on
    # each Drillbit node. (This is not for libraries that are distributed
    # by YARN.) (Place Drill-specific libraries in $DRILL_SITE/lib.)

    library-path: "/opt/libs"
  }

  # Application Master settings. You should not have to change these.

  am: {

    # Heap memory for the AM. Change this only if you run a large cluster and
    # encounter memory errors.

    heap: "450M"

    # Amount of memory to allocate for the YARN container. Adjust this only if
    # you adjust the heap amount above.

    memory-mb: 512

    # Optional label to apply to the AM request. Restricts the AM to run on
    # only nodes with the label. Leave blank unless you need to restrict AM
    # location.

    node-label-expr: "drill-am"
  }

  # Settings for the Application Master's internal web server.

  http: {

    # Listen port. Change this if you run multiple Drill clusters as
    # YARN may allocate two or more AMs on the same YARN worker node.

    port: 8048

    # Turn this on to enable HTTPS for the AM web UI.

    ssl-enabled: true

    # The type of authorization to use for the AM web UI. One of
    # "none" (no authorization, the default), "drill" (use Drill's
    # configured authentication system), or "simple" (use the user name
    # and password defined here.) With Drill authentication enabled, only
    # the user that launched Drill-on-YARN can access the Drill-on-YARN
    # web UI. With simple authentication, then the defined user name and
    # password is the only valid user.

    auth-type: "drill"

    # Set only for the simple auth type.

    user_name: "fred"
    password: "wilma"

    # Set this to some value to protect the management REST calls.
    # When the key is set, each rest call must include the key=value
    # parameter. This is not high security, just a speed-bump to prevent
    # accidental cluster changes.

    rest-key=""
  }

  # Configuration for each Drillbit. Drill requires that all Drillbits use the
  # same configuration.

  drillbit: {

    # The amount of heap memory to allocate to Drill. This is the equivalent of the
    # DRILL_HEAP environment variable in drill-env.sh when launching Drill outside
    # of YARN.

    heap: "4G"

    # The maximum amount of direct memory to allocate to Drill. This is the equivalent of the
    # DRILL_MAX_DIRECT_MEMORY environment variable in drill-env.sh when launching Drill outside
    # of YARN.

    max-direct-memory: "8G"

    # The amount of memory to alocate to the JVM code cache which holds the classes
    # that Drill creates dynamically for each query execution. The default setting is
    # fine for most applications.

    code-cache: "1G"

    # The amount of memory to allocate to each Drill-bit. This is the amount of memory requested
    # from YARN for each drill-bit. This must be sufficient for
    # heap + max-direct-memory + code-cache + JVM overhead (assume 1G)

    memory-mb: 14336

    # The number of YARN virtual cores to allocate to each drill-bit.

    vcores: 4

    # The number of disk channels (or spindles) that Drill typically uses.
    # This value is supported only only selected YARN distributions.

    disks: 3

    # Add to the class path. Use for things such as HBase, etc. which must be installed
    # on every node. (Drill automatically adds its own class path as well as the one
    # provided by YARN.)

    classpath: ""
  }

  # The groups of hosts on which to launch Drill. Each group can be one of several types.
  #
  # Note: This version supports a single group. The example below has multiple groups
  # only to illustrate the various group types.

  cluster: [
    {
      # Arbitrary name, just used for error and statistics reporting.
      # Defaults to "group-" group index, starting at 1.

      name: "group1"

      # Basic group: let YARN pick any available node.

      type: "basic"

      # Number of Drill-bits to run. (Drill-on-YARN runs each on a separate host,
      # so this is also the number of hosts that will run Drill.)

      count: 2
    }
  ]
}