| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| --- |
| - name: Create Metron streaming directories |
| file: path="{{ metron_directory }}/{{ item.name }}" state=directory mode=0755 |
| with_items: |
| - { name: 'lib'} |
| - { name: 'bin'} |
| - { name: 'config'} |
| |
| |
| - name: Copy Metron Solr bundle |
| copy: |
| src: "{{ metron_solr_bundle_path }}" |
| dest: "{{ metron_directory }}" |
| |
| - name: Copy Metron Elasticsearch bundle |
| copy: |
| src: "{{ metron_elasticsearch_bundle_path }}" |
| dest: "{{ metron_directory }}" |
| |
| - name: Copy Metron Topologies bundle |
| copy: |
| src: "{{ metron_topologies_bundle_path }}" |
| dest: "{{ metron_directory }}" |
| |
| - name: Copy Metron DataLoads bundle |
| copy: |
| src: "{{ metron_dataloads_path }}" |
| dest: "{{ metron_directory }}" |
| |
| - name: Unbundle Metron bundles |
| shell: cd {{ metron_directory }} && tar xzvf Metron-Solr*.tar.gz && tar xzvf Metron-Elasticsearch*.tar.gz && tar xzvf Metron-Topologies*.tar.gz && tar xzvf Metron-DataLoads*.tar.gz && rm *.tar.gz |
| |
| - name: Add *-site.xml files to topology jars |
| shell: cd {{ item.config_path }} && jar -uf {{ metron_directory }}/lib/{{ item.jar_name }} {{ item.file_name }} |
| with_items: |
| - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hbase-site.xml" } |
| - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "core-site.xml" } |
| - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hdfs-site.xml" } |
| - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hbase-site.xml" } |
| - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "core-site.xml" } |
| - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hdfs-site.xml" } |
| |
| - name: Get Default mysql passowrd |
| include_vars: "../roles/mysql_server/defaults/main.yml" |
| when: mysql_root_password is undefined |
| |
| - include: hdfs_filesystem.yml |
| run_once: true |
| |
| - include: grok_upload.yml |
| run_once: true |
| |
| - name: Configure Metron Solr topologies |
| lineinfile: > |
| dest={{ metron_solr_properties_config_path }} |
| regexp="{{ item.regexp }}" |
| line="{{ item.line }}" |
| with_items: |
| - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" } |
| - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" } |
| - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" } |
| - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" } |
| - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" } |
| - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" } |
| - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" } |
| - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" } |
| - { regexp: "bolt.hbase.table.name=", line: "bolt.hbase.table.name={{ pcap_hbase_table }}" } |
| - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" } |
| - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" } |
| - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" } |
| - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" } |
| - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" } |
| - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" } |
| - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" } |
| - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" } |
| - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" } |
| - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" } |
| - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" } |
| - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" } |
| |
| - name: Configure Metron Elasticsearch topologies |
| lineinfile: > |
| dest={{ metron_elasticsearch_properties_config_path }} |
| regexp="{{ item.regexp }}" |
| line="{{ item.line }}" |
| with_items: |
| - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" } |
| - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" } |
| - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" } |
| - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" } |
| - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" } |
| - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" } |
| - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" } |
| - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" } |
| - { regexp: "bolt.hbase.table.name=", line: "bolt.hbase.table.name={{ pcap_hbase_table }}" } |
| - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" } |
| - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" } |
| - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" } |
| - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" } |
| - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" } |
| - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" } |
| - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" } |
| - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" } |
| - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" } |
| - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" } |
| - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" } |
| - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" } |
| |
| - include: source_config.yml |
| run_once: true |
| |
| - include: threat_intel.yml |
| run_once: true |
| when: threat_intel_bulk_load == True |
| |
| - include: metron_topology.yml |
| |
| - include: hdfs_purge.yml |
| |
| - include: es_purge.yml |
