| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| --- |
| |
| hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER] |
| app_timeline_server: [APP_TIMELINE_SERVER] |
| hadoop_slave: [DATANODE, NODEMANAGER] |
| spark_master: [SPARK_JOBHISTORYSERVER] |
| storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER] |
| storm_slave: [SUPERVISOR] |
| kafka_broker: [KAFKA_BROKER] |
| zookeeper_master: [ZOOKEEPER_SERVER] |
| hbase_master: [HBASE_MASTER] |
| hbase_slave: [HBASE_REGIONSERVER] |
| hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT] |
| es_master: [ES_MASTER] |
| es_slave: [ES_SLAVE] |
| kibana_master: [KIBANA_MASTER] |
| metron_indexing: [METRON_INDEXING] |
| metron_enrichment_master : [METRON_ENRICHMENT_MASTER] |
| metron_parsers : [METRON_PARSERS] |
| metron_rest: [METRON_REST] |
| metron_management_ui: [METRON_MANAGEMENT_UI] |
| |
| master_1_components: "{{ hadoop_master | union(hadoop_clients) | union(es_slave) }}" |
| master_1_host: |
| - "{{groups.ambari_slave[0]}}" |
| master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) | union(app_timeline_server) | union(es_slave) }}" |
| master_2_host: |
| - "{{groups.ambari_slave[1]}}" |
| metron_components: > |
| {{ metron_indexing | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(hadoop_slave) | union(storm_slave) | |
| union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }} |
| metron_host: |
| - "{{ groups.metron[0] }}" |
| web_components: "{{ kibana_master | union(es_master) }}" |
| web_host: |
| - "{{ groups.web[0] }}" |
| slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}" |
| |
| cluster_name: "metron" |
| blueprint_name: "metron_blueprint" |
| |
| configurations: |
| - zoo.cfg: |
| dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}' |
| - hadoop-env: |
| namenode_heapsize: 2048 |
| dtnode_heapsize: 1024 |
| - hbase-env: |
| hbase_regionserver_heapsize: 1024 |
| hbase_master_heapsize: 1024 |
| - hdfs-site: |
| dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}' |
| dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}' |
| dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}' |
| dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}' |
| - mapred-site: |
| mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}' |
| mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}' |
| mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}' |
| - yarn-site: |
| yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}' |
| yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}' |
| yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}' |
| yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}' |
| yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}' |
| - storm-site: |
| supervisor.slots.ports: "[6700, 6701, 6702, 6703]" |
| storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}' |
| topology.classpath: '{{ topology_classpath }}' |
| - kafka-broker: |
| log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}' |
| - metron-env: |
| parsers: "bro,snort,yaf" |
| - elastic-site: |
| index_number_of_shards: 2 |
| index_number_of_replicas: 1 |
| zen_discovery_ping_unicast_hosts: "[ {{ groups.web[0] }}, {{ groups.search | join(', ') }} ]" |
| gateway_recover_after_data_nodes: 1 |
| network_host: "[ _local_, {{ elasticsearch_network_interface }} ]" |
| |
| required_configurations: |
| - metron-env: |
| storm_rest_addr: "http://{{ groups.ambari_slave[1] }}:8744" |
| es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}" |
| zeppelin_server_url: "{{ groups.zeppelin[0] }}" |
| metron_jdbc_driver: "org.h2.Driver" |
| metron_jdbc_url: "jdbc:h2:file:~/metrondb" |
| metron_jdbc_username: "root" |
| metron_jdbc_password: "root" |
| metron_jdbc_platform: "h2" |
| - kibana-env: |
| kibana_pid_dir: /var/run/kibana |
| kibana_es_url: http://{{ groups.web[0] }}:9200 |
| kibana_log_dir: /var/log/kibana |
| kibana_server_port: 5000 |
| kibana_default_application: "dashboard/Metron-Dashboard" |
| |
| blueprint: |
| stack_name: HDP |
| stack_version: "{{ hdp_stack }}" |
| required_configurations: "{{ required_configurations }}" |
| groups: |
| - name : master_1 |
| cardinality: 1 |
| configuration: [] # configuration not yet implemented |
| components: "{{ master_1_components }}" |
| hosts: "{{ master_1_host }}" |
| - name : master_2 |
| cardinality: 1 |
| configuration: [] # configuration not yet implemented |
| components: "{{ master_2_components }}" |
| hosts: "{{ master_2_host }}" |
| - name : metron |
| cardinality: 1 |
| configuration: [] # configuration not yet implemented |
| components: "{{ metron_components }}" |
| hosts: "{{ metron_host }}" |
| - name : web |
| cardinality: 1 |
| configuration: [] # configuration not yet implemented |
| components: "{{ web_components }}" |
| hosts: "{{ web_host }}" |
| - name: slaves |
| cardinality: 1+ |
| configuration: [] # configuration not yet implemented |
| components: "{{ slave_components }}" |
| hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) | difference(groups.metron[0]) | difference(groups.web[0]) }}" |