metron-deployment/roles/ambari_config/vars/small_cluster.yml

#
#  Licensed to the Apache Software Foundation (ASF) under one or more
#  contributor license agreements.  See the NOTICE file distributed with
#  this work for additional information regarding copyright ownership.
#  The ASF licenses this file to You under the Apache License, Version 2.0
#  (the "License"); you may not use this file except in compliance with
#  the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
---

hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER]
app_timeline_server: [APP_TIMELINE_SERVER]
hadoop_slave: [DATANODE, NODEMANAGER]
spark_master: [SPARK_JOBHISTORYSERVER]
storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER]
storm_slave: [SUPERVISOR]
kafka_broker: [KAFKA_BROKER]
zookeeper_master: [ZOOKEEPER_SERVER]
hbase_master: [HBASE_MASTER]
hbase_slave: [HBASE_REGIONSERVER]
hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT]
es_master: [ES_MASTER]
es_slave: [ES_SLAVE]
kibana_master: [KIBANA_MASTER]
metron_indexing: [METRON_INDEXING]
metron_profiler: [METRON_PROFILER]
metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
metron_parsers : [METRON_PARSERS]
metron_rest: [METRON_REST]
metron_management_ui: [METRON_MANAGEMENT_UI]

master_1_components: "{{ hadoop_master | union(hadoop_clients) | union(es_slave) }}"
master_1_host:
  - "{{groups.ambari_slave[0]}}"
master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) | union(app_timeline_server) | union(es_slave) }}"
master_2_host:
  - "{{groups.ambari_slave[1]}}"
metron_components: >
    {{ metron_indexing | union(metron_profiler) | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(hadoop_slave) | union(storm_slave) |
    union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
metron_host:
  - "{{ groups.metron[0] }}"
web_components: "{{ kibana_master | union(es_master) }}"
web_host:
  - "{{ groups.web[0] }}"
slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}"

cluster_name: "metron"
blueprint_name: "metron_blueprint"

configurations:
  - zoo.cfg:
      dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
  - hadoop-env:
      namenode_heapsize: 2048
      dtnode_heapsize: 1024
  - hbase-env:
      hbase_regionserver_heapsize: 1024
      hbase_master_heapsize: 1024
  - hdfs-site:
      dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}'
      dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}'
      dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
      dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}'
  - mapred-site:
      mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}'
      mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}'
      mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}'
  - yarn-site:
      yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}'
      yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}'
      yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}'
      yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}'
      yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
  - storm-site:
      supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704]"
      storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
      topology.classpath: '{{ topology_classpath }}'
  - kafka-broker:
      log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}'
  - metron-env:
      parsers: "bro,snort,yaf"
  - elastic-site:
      index_number_of_shards: 2
      index_number_of_replicas: 1
      zen_discovery_ping_unicast_hosts: "[ {{ groups.web[0] }}, {{ groups.search | join(', ') }} ]"
      gateway_recover_after_data_nodes: 1
      network_host: "[ _local_, {{ elasticsearch_network_interface }} ]"

required_configurations:
  - metron-env:
      storm_rest_addr: "http://{{ groups.ambari_slave[1] }}:8744"
      es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}"
      zeppelin_server_url: "{{ groups.zeppelin[0] }}"
  - metron-rest-env:
      metron_jdbc_driver: "org.h2.Driver"
      metron_jdbc_url: "jdbc:h2:file:~/metrondb"
      metron_jdbc_username: "root"
      metron_jdbc_password: "root"
      metron_jdbc_platform: "h2"
  - kibana-env:
      kibana_pid_dir: /var/run/kibana
      kibana_es_url: http://{{ groups.web[0] }}:9200
      kibana_log_dir: /var/log/kibana
      kibana_server_port: 5000
      kibana_default_application: "dashboard/Metron-Dashboard"

blueprint:
  stack_name: HDP
  stack_version: "{{ hdp_stack }}"
  required_configurations: "{{ required_configurations }}"
  groups:
    - name : master_1
      cardinality: 1
      configuration: []  # configuration not yet implemented
      components: "{{ master_1_components }}"
      hosts: "{{ master_1_host }}"
    - name : master_2
      cardinality: 1
      configuration: []  # configuration not yet implemented
      components: "{{ master_2_components }}"
      hosts: "{{ master_2_host }}"
    - name : metron
      cardinality: 1
      configuration: []  # configuration not yet implemented
      components: "{{ metron_components }}"
      hosts: "{{ metron_host }}"
    - name : web
      cardinality: 1
      configuration: []  # configuration not yet implemented
      components: "{{ web_components }}"
      hosts: "{{ web_host }}"
    - name: slaves
      cardinality: 1+
      configuration: []  # configuration not yet implemented
      components: "{{ slave_components }}"
      hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) | difference(groups.metron[0]) | difference(groups.web[0]) }}"