metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml

#
#  Licensed to the Apache Software Foundation (ASF) under one or more
#  contributor license agreements.  See the NOTICE file distributed with
#  this work for additional information regarding copyright ownership.
#  The ASF licenses this file to You under the Apache License, Version 2.0
#  (the "License"); you may not use this file except in compliance with
#  the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
---

hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER]
app_timeline_server: [APP_TIMELINE_SERVER]
hadoop_slave: [DATANODE, NODEMANAGER]
spark_master: [SPARK_JOBHISTORYSERVER]
storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER]
storm_slave: [SUPERVISOR]
kafka_broker: [KAFKA_BROKER]
zookeeper_master: [ZOOKEEPER_SERVER]
hbase_master: [HBASE_MASTER]
hbase_slave: [HBASE_REGIONSERVER]
hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT]
es_master: [ES_MASTER]
es_slave: [ES_SLAVE]
kibana_master: [KIBANA_MASTER]
metron_indexing: [METRON_INDEXING]
metron_profiler: [METRON_PROFILER]
metron_pcap: [METRON_PCAP]
metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
metron_parsers : [METRON_PARSERS]
metron_rest: [METRON_REST]
metron_management_ui: [METRON_MANAGEMENT_UI]
metron_alerts_ui: [METRON_ALERTS_UI]

master_1_components: "{{ hadoop_master | union(hadoop_clients) | union(es_slave) }}"
master_1_host:
  - "{{groups.ambari_slave[0]}}"
master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) | union(app_timeline_server) | union(es_slave) }}"
master_2_host:
  - "{{groups.ambari_slave[1]}}"
metron_components: >
    {{ metron_indexing | union(metron_profiler) | union(metron_pcap) |union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) | union(storm_slave) |
    union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
metron_host:
  - "{{ groups.metron[0] }}"
web_components: "{{ kibana_master | union(es_master) }}"
web_host:
  - "{{ groups.web[0] }}"
slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}"

cluster_name: "metron"
blueprint_name: "metron_blueprint"

configurations:
  - zoo.cfg:
      dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
  - hadoop-env:
      namenode_heapsize: 2048
      dtnode_heapsize: 1024
  - hbase-env:
      hbase_regionserver_heapsize: 1024
      hbase_master_heapsize: 1024
  - hdfs-site:
      dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}'
      dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}'
      dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
      dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}'
  - mapred-site:
      mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}'
      mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}'
      mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}'
  - yarn-site:
      yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}'
      yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}'
      yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}'
      yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}'
      yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
  - storm-site:
      supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]"
      storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
      topology.classpath: '{{ topology_classpath }}'
      # Storm expects ambari metrics to be available in 2.6.  We do *not* install ambari metrics in full-dev, so we need to revert to the old consumer
      storm.cluster.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer"}]'
      topology.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer", "parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency", "__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count", "__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$", "GC/.+\\.count$", "GC/.+\\.timeMs$"]}]'
      # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to use and thus that must be removed from nimbus, supervisor and worker properties 
      nimbus.childopts: '-Xmx1024m _JAAS_PLACEHOLDER'
      supervisor.childopts: '-Xmx256m _JAAS_PLACEHOLDER'
      worker.childopts: "-Xmx768m _JAAS_PLACEHOLDER"
  - kafka-broker:
      log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}'
  - metron-rest-env:
      metron_spring_profiles_active: "dev"
  - metron-env:
      parsers: "bro,snort,yaf"
  - elastic-site:
      index_number_of_shards: 2
      index_number_of_replicas: 1
      zen_discovery_ping_unicast_hosts: "[ {{ groups.web[0] }}, {{ groups.search | join(', ') }} ]"
      gateway_recover_after_data_nodes: 1
      network_host: "[ _local_, {{ elasticsearch_network_interface }} ]"

required_configurations:
  - metron-env:
      storm_rest_addr: "http://{{ groups.ambari_slave[1] }}:8744"
      es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}"
      zeppelin_server_url: "{{ groups.zeppelin[0] }}"
  - metron-rest-env:
      metron_jdbc_driver: "org.h2.Driver"
      metron_jdbc_url: "jdbc:h2:file:~/metrondb"
      metron_jdbc_username: "root"
      metron_jdbc_password: "root"
      metron_jdbc_platform: "h2"
  - kibana-env:
      kibana_pid_dir: /var/run/kibana
      kibana_es_url: http://{{ groups.web[0] }}:9200
      kibana_log_dir: /var/log/kibana
      kibana_server_port: 5000
      kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9"
      kibana_server_host: 0.0.0.0

blueprint:
  stack_name: HDP
  stack_version: "{{ hdp_stack }}"
  required_configurations: "{{ required_configurations }}"
  groups:
    - name : master_1
      cardinality: 1
      configuration: []  # configuration not yet implemented
      components: "{{ master_1_components }}"
      hosts: "{{ master_1_host }}"
    - name : master_2
      cardinality: 1
      configuration: []  # configuration not yet implemented
      components: "{{ master_2_components }}"
      hosts: "{{ master_2_host }}"
    - name : metron
      cardinality: 1
      configuration: []  # configuration not yet implemented
      components: "{{ metron_components }}"
      hosts: "{{ metron_host }}"
    - name : web
      cardinality: 1
      configuration: []  # configuration not yet implemented
      components: "{{ web_components }}"
      hosts: "{{ web_host }}"
    - name: slaves
      cardinality: 1+
      configuration: []  # configuration not yet implemented
      components: "{{ slave_components }}"
      hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) | difference(groups.metron[0]) | difference(groups.web[0]) }}"