METRON-152 Add support for ES 2.3.x and Kibana 4.5.0 (dlyle65535 via nickwallen) closes apache/incubator-metron#135
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index a505588..0734ec5 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -20,7 +20,7 @@
- mysql
- elasticsearch
- pcap-service
- - metron-ui
+ - kibana
- yaf
- snort
- snort-logs
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
index 2d4dc7e..0ffac8b 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -20,7 +20,7 @@
- mysql
- elasticsearch
- pcap-service
- - metron-ui
+ - kibana
- yaf
- snort
- snort-logs
diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml
index 42bbeb1..69f35f5 100644
--- a/metron-deployment/playbooks/metron_install.yml
+++ b/metron-deployment/playbooks/metron_install.yml
@@ -106,7 +106,7 @@
- hosts: web
become: true
roles:
- - { role: metron_ui, when: install_elasticsearch | default(True) == True }
+ - { role: kibana, when: install_elasticsearch | default(True) == True }
tags:
- web
diff --git a/metron-deployment/roles/elasticsearch/files/bro_index.template b/metron-deployment/roles/elasticsearch/files/bro_index.template
new file mode 100644
index 0000000..4da5110
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/files/bro_index.template
@@ -0,0 +1,12 @@
+{
+ "template" : "bro_index*",
+ "mappings" : {
+ "bro_doc" : {
+ "_timestamp" : { "enabled" : true },
+ "properties": {
+ "timestamp":{"type":"date","format":"epoch_millis"},
+ "enrichments:geo:ip_dst_addr:location_point" : { "type" : "geo_point" }
+ }
+ }
+ }
+}
diff --git a/metron-deployment/roles/elasticsearch/files/elasticsearch.repo b/metron-deployment/roles/elasticsearch/files/elasticsearch.repo
index f033ced..51dba70 100644
--- a/metron-deployment/roles/elasticsearch/files/elasticsearch.repo
+++ b/metron-deployment/roles/elasticsearch/files/elasticsearch.repo
@@ -15,9 +15,10 @@
# limitations under the License.
#
-[elasticsearch-1.7]
-name=Elasticsearch repository for 1.7.x packages
-baseurl=http://packages.elastic.co/elasticsearch/1.7/centos
+[elasticsearch-2.x]
+name=Elasticsearch repository for 2.x packages
+baseurl=https://packages.elastic.co/elasticsearch/2.x/centos
gpgcheck=1
-gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
+gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
+
diff --git a/metron-deployment/roles/elasticsearch/files/snort_index.template b/metron-deployment/roles/elasticsearch/files/snort_index.template
new file mode 100644
index 0000000..01118ba
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/files/snort_index.template
@@ -0,0 +1,19 @@
+{
+ "template": "snort_index*",
+ "mappings": {
+ "snort_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "enrichments:geo:ip_dst_addr:location_point": {
+ "type": "geo_point"
+ }
+ }
+ }
+ }
+}
diff --git a/metron-deployment/roles/elasticsearch/files/yaf_index.template b/metron-deployment/roles/elasticsearch/files/yaf_index.template
index 15ff6f8..442321e 100644
--- a/metron-deployment/roles/elasticsearch/files/yaf_index.template
+++ b/metron-deployment/roles/elasticsearch/files/yaf_index.template
@@ -2,12 +2,15 @@
"template": "yaf_index*",
"mappings": {
"yaf_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
"properties": {
"timestamp": {
"type": "date",
- "format": "dateOptionalTime"
+ "format": "epoch_millis"
},
- "location_point": {
+ "enrichments:geo:ip_dst_addr:location_point": {
"type": "geo_point"
},
"end-time": {
diff --git a/metron-deployment/roles/elasticsearch/tasks/create-indices.yml b/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
index d91d220..0f27351 100644
--- a/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
+++ b/metron-deployment/roles/elasticsearch/tasks/create-indices.yml
@@ -38,11 +38,14 @@
- name: Add Elasticsearch templates for topologies
uri:
- url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/template_yaf"
- method: POST
- body: "{{ lookup('file','yaf_index.template') }}"
+ url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/template_{{ item.sensor }}"
+ method: PUT
+ body: "{{ item.file }} | to_json "
status_code: 200
- body_format: json
+ with_items:
+ - { sensor: bro, file: "{{ lookup('file','bro_index.template') }}" }
+ - { sensor: yaf, file: "{{ lookup('file','yaf_index.template') }}" }
+ - { sensor: snort, file: "{{ lookup('file','snort_index.template') }}" }
- name: Stop Elasticsearch
service: name=elasticsearch state=stopped
diff --git a/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml b/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml
index 41b7f37..b5df7b9 100644
--- a/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml
+++ b/metron-deployment/roles/elasticsearch/tasks/elasticsearch.yml
@@ -46,16 +46,28 @@
with_items:
- '{{ elasticsearch_data_dir.split(",") }}'
+- name: Configure Elasticsearch - remove stale entries
+ lineinfile: >
+ dest=/etc/elasticsearch/elasticsearch.yml
+ regexp="{{ item }}"
+ state=absent
+ with_items:
+ - '^cluster\.name:'
+ - '^network\.host:'
+ - '^discovery\.zen\.ping\.unicast\.hosts:'
+ - '^path\.data'
+
- name: Configure Elasticsearch
lineinfile: >
dest=/etc/elasticsearch/elasticsearch.yml
regexp="{{ item.regexp }}"
+ insertafter="{{ item.regexp }}"
line="{{ item.line }}"
state=present
with_items:
- - { regexp: '#cluster\.name', line: 'cluster.name: metron' }
- - { regexp: '#network\.host:', line: 'network.host: _{{
+ - { regexp: '^# *cluster\.name:', line: 'cluster.name: metron' }
+ - { regexp: '^# *network\.host:', line: 'network.host: _{{
elasticsearch_network_interface }}:ipv4_' }
- - { regexp: '#discovery\.zen\.ping\.unicast\.hosts',
+ - { regexp: '^# *discovery\.zen\.ping\.unicast\.hosts:',
line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'}
- - { regexp: '#path\.data', line: 'path.data: {{ elasticsearch_data_dir }}' }
+ - { regexp: '^# *path\.data', line: 'path.data: {{ elasticsearch_data_dir }}' }
diff --git a/metron-deployment/roles/metron_ui/tasks/main.yml b/metron-deployment/roles/kibana/defaults/main.yml
similarity index 85%
copy from metron-deployment/roles/metron_ui/tasks/main.yml
copy to metron-deployment/roles/kibana/defaults/main.yml
index f7bb733..073d4b8 100644
--- a/metron-deployment/roles/metron_ui/tasks/main.yml
+++ b/metron-deployment/roles/kibana/defaults/main.yml
@@ -14,7 +14,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
----
-- include: dependencies.yml
-- include: copy-source.yml
-- include: metron-ui.yml
+kibana_repo_url: http://packages.elastic.co/kibana/4.5/centos
+kibana_repo_key_url: http://packages.elastic.co/GPG-KEY-elasticsearch
diff --git a/metron-deployment/roles/metron_ui/defaults/main.yml b/metron-deployment/roles/kibana/tasks/dependencies.yml
similarity index 77%
rename from metron-deployment/roles/metron_ui/defaults/main.yml
rename to metron-deployment/roles/kibana/tasks/dependencies.yml
index 23aed40..7450665 100644
--- a/metron-deployment/roles/metron_ui/defaults/main.yml
+++ b/metron-deployment/roles/kibana/tasks/dependencies.yml
@@ -15,7 +15,13 @@
# limitations under the License.
#
---
-metron_version: 0.1BETA
-metron_directory: /usr/metron/{{ metron_version }}
-metron_ui_directory: "{{ metron_directory }}/metron-ui"
-metron_temp_archive: /tmp/metron-ui.tar.gz
+- name: Load Elastic Public Signing Key
+ rpm_key:
+ state: present
+ key: "{{ kibana_repo_key_url }}"
+
+- name: Create kibana.repo
+ template:
+ src: kibana.repo.template
+ dest: /etc/yum.repos.d/kibana.repo
+ mode: 0644
diff --git a/metron-deployment/roles/kibana/tasks/install_kibana.yml b/metron-deployment/roles/kibana/tasks/install_kibana.yml
new file mode 100644
index 0000000..7df077e
--- /dev/null
+++ b/metron-deployment/roles/kibana/tasks/install_kibana.yml
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install Kibana
+ yum:
+ name: kibana
+ state: installed
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- name: Configure Kibana - remove stale entries
+ lineinfile: >
+ dest=/opt/kibana/config/kibana.yml
+ regexp="{{ item }}"
+ state=absent
+ with_items:
+ - '^server\.port:'
+ - '^elasticsearch\.url:'
+
+- name: Configure Kibana
+ lineinfile: >
+ dest=/opt/kibana/config/kibana.yml
+ regexp="{{ item.regexp }}"
+ insertafter="{{ item.regexp }}"
+ line="{{ item.line }}"
+ state=present
+ with_items:
+ - { regexp: '^# *server\.port:', line: 'server.port: 5000' }
+ - { regexp: '^# *elasticsearch\.url:', line: 'elasticsearch.url: http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}' }
diff --git a/metron-deployment/roles/metron_ui/tasks/main.yml b/metron-deployment/roles/kibana/tasks/main.yml
similarity index 93%
rename from metron-deployment/roles/metron_ui/tasks/main.yml
rename to metron-deployment/roles/kibana/tasks/main.yml
index f7bb733..280223b 100644
--- a/metron-deployment/roles/metron_ui/tasks/main.yml
+++ b/metron-deployment/roles/kibana/tasks/main.yml
@@ -16,5 +16,5 @@
#
---
- include: dependencies.yml
-- include: copy-source.yml
-- include: metron-ui.yml
+- include: install_kibana.yml
+
diff --git a/metron-deployment/roles/metron_ui/tasks/main.yml b/metron-deployment/roles/kibana/templates/kibana.repo.template
similarity index 85%
copy from metron-deployment/roles/metron_ui/tasks/main.yml
copy to metron-deployment/roles/kibana/templates/kibana.repo.template
index f7bb733..d5c096f 100644
--- a/metron-deployment/roles/metron_ui/tasks/main.yml
+++ b/metron-deployment/roles/kibana/templates/kibana.repo.template
@@ -14,7 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
----
-- include: dependencies.yml
-- include: copy-source.yml
-- include: metron-ui.yml
+[kibana]
+name=Kibana installation repository
+baseurl={{ kibana_repo_url }}
+gpgcheck=1
+gpgkey={{ kibana_repo_key_url }}
+enabled=1
diff --git a/metron-deployment/roles/metron_streaming/files/yaf_index.template b/metron-deployment/roles/metron_streaming/files/yaf_index.template
deleted file mode 100644
index c8c1702..0000000
--- a/metron-deployment/roles/metron_streaming/files/yaf_index.template
+++ /dev/null
@@ -1,36 +0,0 @@
-{
- "template" : "yaf_index*",
- "mappings" : {
- "yaf_doc" : {
- "properties": {
- "message": {
- "properties": {
- "@timestamp":{"type":"date","format":"dateOptionalTime"},
- "end-time":{"type":"string"},
- "duration":{"type":"string"},
- "rtt":{"type":"string"},
- "proto":{"type":"string"},
- "sip":{"type":"string"},
- "sp":{"type":"string"},
- "dip":{"type":"string"},
- "dp":{"type":"string"},
- "iflags":{"type":"string"},
- "uflags":{"type":"string"},
- "riflags":{"type":"string"},
- "ruflags":{"type":"string"},
- "isn":{"type":"string"},
- "risn":{"type":"string"},
- "tag":{"type":"string"},
- "rtag":{"type":"string"},
- "pkt":{"type":"string"},
- "oct":{"type":"string"},
- "rpkt":{"type":"string"},
- "roct":{"type":"string"},
- "app":{"type":"string"},
- "end-reason":{"type":"string"}
- }
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/metron-deployment/roles/metron_ui/README.md b/metron-deployment/roles/metron_ui/README.md
deleted file mode 100644
index 54eb6e4..0000000
--- a/metron-deployment/roles/metron_ui/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-Metron Deployment - Metron UI
-=============================
-
-This project deploys all of the necessary components to run the Metron UI. This is installed as a SysV service for management purposes.
-
-Getting Started
----------------
-
-```
-service pm2-init.sh start
-service pm2-init.sh stop
-service pm2-init.sh restart
-service pm2-init.sh status
-```
diff --git a/metron-deployment/roles/metron_ui/tasks/copy-source.yml b/metron-deployment/roles/metron_ui/tasks/copy-source.yml
deleted file mode 100644
index 703b7f9..0000000
--- a/metron-deployment/roles/metron_ui/tasks/copy-source.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Verify {{ metron_ui_directory }} exists
- file:
- path: "{{ metron_ui_directory }}"
- state: directory
- mode: 0755
- owner: root
- group: root
-
-- name: Archive metron-ui on localhost
- shell: tar --exclude='./node_modules' -czf {{ metron_temp_archive }} .
- args:
- chdir: "{{ playbook_dir }}/../../metron-ui"
- creates: "{{ metron_temp_archive }}"
- warn: false #Warns to use unarchive - unarchive does not archive
- become: false
- delegate_to: localhost
- run_once: true
-
-- name: Extract metron-ui tarball
- unarchive:
- src: "{{ metron_temp_archive }}"
- dest: "{{ metron_ui_directory }}"
- creates: "{{ metron_ui_directory}}/config"
-
-- name: Delete {{ metron_temp_archive }}
- local_action: file path="{{ metron_temp_archive }}" state=absent
- become: false
- run_once: true
-
diff --git a/metron-deployment/roles/metron_ui/tasks/dependencies.yml b/metron-deployment/roles/metron_ui/tasks/dependencies.yml
deleted file mode 100644
index fd849f7..0000000
--- a/metron-deployment/roles/metron_ui/tasks/dependencies.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install yum repositories
- yum: name=epel-release update_cache=yes
-
-- name: Install Metron UI dependencies
- yum:
- pkg: "{{ item }}"
- state: installed
- with_items:
- - libpcap-devel
- - wireshark
- - nodejs
- - npm
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
diff --git a/metron-deployment/roles/metron_ui/tasks/metron-ui.yml b/metron-deployment/roles/metron_ui/tasks/metron-ui.yml
deleted file mode 100644
index 1e4d30a..0000000
--- a/metron-deployment/roles/metron_ui/tasks/metron-ui.yml
+++ /dev/null
@@ -1,57 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Configure Metron UI
- lineinfile:
- dest="{{ metron_ui_directory }}/config.json"
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- state=present
- with_items:
- - { regexp: '"elasticsearch":', line: '"elasticsearch": { "url": "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}" },' }
- - { regexp: '"pcap":', line: ' "pcap": { "url": "http://{{ groups.pcap_server[0] }}:{{ pcapservice_port }}/pcapGetter","mock": false }' }
-
-- name: Install Node dependencies
- npm:
- name: pm2
- path: "{{ metron_ui_directory }}"
- global: true
-
-- name: Install Metron UI
- npm:
- path: "{{ metron_ui_directory }}"
- production: no
-
-- name: Ensure Metron UI is stopped before installing service
- shell: pm2 stop all
- args:
- creates: /etc/init.d/pm2-init.sh
- register: pm2_result
- failed_when: pm2_result.rc != 0 and ("No process found" not in pm2_result.stderr)
-
-- name: Configure Metron UI as a service
- shell: "{{ item }}"
- args:
- creates: /etc/init.d/pm2-init.sh
- with_items:
- - "pm2 start {{ metron_ui_directory }}/lib/metron-ui.js --name metron"
- - pm2 save
- - pm2 startup centos
- - su -c 'chmod +x /etc/init.d/pm2-init.sh'
-
-- name: Stop Metron UI
- service: name=pm2-init.sh state=stopped
diff --git a/metron-deployment/roles/monit/tasks/monit-definitions.yml b/metron-deployment/roles/monit/tasks/monit-definitions.yml
index 18ea0b0..3778e14 100644
--- a/metron-deployment/roles/monit/tasks/monit-definitions.yml
+++ b/metron-deployment/roles/monit/tasks/monit-definitions.yml
@@ -28,8 +28,8 @@
template: src=monit/enrichment-elasticsearch.monit dest={{ monit_config_home }}/enrichment-elasticsearch.monit
when: ("enrichment" in group_names) and (install_elasticsearch | default(True))
-- name: Create monit definition for metron-ui
- template: src=monit/metron-ui.monit dest={{ monit_config_home }}/metron-ui.monit
+- name: Create monit definition for kibana
+ template: src=monit/kibana.monit dest={{ monit_config_home }}/kibana.monit
when: ("web" in group_names) and (install_elasticsearch | default(True))
- name: Create monit definition for mysql
diff --git a/metron-deployment/roles/monit/templates/monit/metron-ui.monit b/metron-deployment/roles/monit/templates/monit/kibana.monit
similarity index 81%
rename from metron-deployment/roles/monit/templates/monit/metron-ui.monit
rename to metron-deployment/roles/monit/templates/monit/kibana.monit
index 1b7e9cc..41b4cb9 100644
--- a/metron-deployment/roles/monit/templates/monit/metron-ui.monit
+++ b/metron-deployment/roles/monit/templates/monit/kibana.monit
@@ -14,9 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-check process metron-ui matching "node /usr/metron/{{ metron_version }}/metron-ui/lib/metron-ui.js"
- start program = "/etc/init.d/pm2-init.sh start"
- stop program = "/etc/init.d/pm2-init.sh stop"
+check process kibana matching "/opt/kibana/bin/../node/bin/node /opt/kibana/bin/../src/cli"
+ start program = "/etc/init.d/kibana start"
+ stop program = "/etc/init.d/kibana stop"
if does not exist then restart
group web
group metron
diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml
new file mode 100644
index 0000000..2a85e7a
--- /dev/null
+++ b/metron-platform/elasticsearch-shaded/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>metron-platform</artifactId>
+ <groupId>org.apache.metron</groupId>
+ <version>0.1BETA</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>elasticsearch-shaded</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>18.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${global_elasticsearch_version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.metron.guava.elasticsearch-shaded</shadedPattern>
+ </relocation>
+ </relocations>
+ <artifactSet>
+ <excludes>
+ <exclude>storm:storm-core:*</exclude>
+ <exclude>storm:storm-lib:*</exclude>
+ <exclude>org.slf4j.impl*</exclude>
+ <exclude>org.slf4j:slf4j-log4j*</exclude>
+ </excludes>
+ </artifactSet>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resource>.yaml</resource>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 9d439e0..172d387 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -151,11 +151,20 @@
</exclusions>
</dependency>
<dependency>
- <groupId>com.github.fge</groupId>
- <artifactId>json-schema-validator</artifactId>
- <version>${global_json_schema_validator_version}</version>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${global_jackson_version}</version>
</dependency>
-
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${global_jackson_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${global_jackson_version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/interfaces/FieldNameConverter.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/interfaces/FieldNameConverter.java
new file mode 100644
index 0000000..92e7ec6
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/interfaces/FieldNameConverter.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.common.interfaces;
+
+public interface FieldNameConverter {
+
+ String convert(String originalField);
+
+}
diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml
index 941c265..68ffc65 100644
--- a/metron-platform/metron-data-management/pom.xml
+++ b/metron-platform/metron-data-management/pom.xml
@@ -25,6 +25,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<httpcore.version>4.3.2</httpcore.version>
+ <lucene.test.version>5.5.0</lucene.test.version>
</properties>
<dependencies>
<dependency>
@@ -167,13 +168,13 @@
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-test-framework</artifactId>
- <version>4.10.4</version>
+ <version>${lucene.test.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
- <version>4.10.4</version>
+ <version>${lucene.test.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java
index f0a4d3b..d424004 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java
@@ -24,12 +24,14 @@
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.metron.common.configuration.Configuration;
import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.Date;
@@ -91,11 +93,15 @@
configuration.update();
Map<String, Object> globalConfiguration = configuration.getGlobalConfig();
- ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder();
- builder.put("cluster.name", globalConfiguration.get("es.clustername"));
- builder.put("curatorFramework.transport.ping_timeout","500s");
- client = new TransportClient(builder.build())
- .addTransportAddress(new InetSocketTransportAddress(globalConfiguration.get("es.ip").toString(), Integer.parseInt(globalConfiguration.get("es.port").toString())));
+
+ Settings.Builder settingsBuilder = Settings.settingsBuilder();
+ settingsBuilder.put("cluster.name", globalConfiguration.get("es.clustername"));
+ settingsBuilder.put("curatorFramework.transport.ping_timeout","500s");
+ Settings settings = settingsBuilder.build();
+ client = TransportClient.builder().settings(settings).build()
+ .addTransportAddress(
+ new InetSocketTransportAddress(InetAddress.getByName(globalConfiguration.get("es.ip").toString()), Integer.parseInt(globalConfiguration.get("es.port").toString()) )
+ );
DataPruner pruner = new ElasticsearchDataPruner(startDate, numDays, configuration, client, indexPrefix);
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerIntegrationTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerIntegrationTest.java
deleted file mode 100644
index 08f95b9..0000000
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerIntegrationTest.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.dataloads.bulk;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.metron.TestConstants;
-import org.apache.metron.common.configuration.Configuration;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.IndexMissingException;
-import org.elasticsearch.test.ElasticsearchIntegrationTest;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
-
-@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE, numDataNodes = 1, numClientNodes = 0)
-public class ElasticsearchDataPrunerIntegrationTest extends ElasticsearchIntegrationTest {
-
- private static File dataPath = new File("./target/elasticsearch-test");
- private Date testingDate;
- private Date yesterday = new Date();
- private DateFormat dateFormat = new SimpleDateFormat("yyyy.MM.dd.HH");
- private Configuration configuration;
-
- @BeforeClass
- public static void setupClass() throws Exception {
-
- if (dataPath.isDirectory()) {
- FileUtils.deleteDirectory(dataPath);
- }
-
- if (!dataPath.mkdirs()) {
- throw new RuntimeException("Couldn't create dataPath at: " + dataPath.getAbsolutePath());
- }
-
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
-
- if (dataPath.isDirectory()) {
- FileUtils.deleteDirectory(dataPath);
- }
-
- }
-
- @Before
- public void setUp() throws Exception {
-
- super.setUp();
- ensureGreen();
-
- TimeZone timeZone = TimeZone.getTimeZone("UTC");
- Calendar calendar = Calendar.getInstance(timeZone);
- calendar.set(Calendar.HOUR_OF_DAY,0);
- calendar.set(Calendar.MINUTE,0);
- calendar.set(Calendar.SECOND,0);
- testingDate = calendar.getTime();
- yesterday.setTime(testingDate.getTime() - TimeUnit.DAYS.toMillis(1));
- dateFormat.setTimeZone(timeZone);
-
- File resourceFile = new File(TestConstants.SAMPLE_CONFIG_PATH);
- Path resourcePath = Paths.get(resourceFile.getCanonicalPath());
-
- configuration = new Configuration(resourcePath);
- }
-
- @Test(expected = IndexMissingException.class)
- public void testWillThrowOnMissingIndex() throws Exception {
-
- ElasticsearchDataPruner pruner = new ElasticsearchDataPruner(yesterday, 30, configuration,client(), "*");
- pruner.deleteIndex(admin(), "baz");
-
- }
-
- @Test
- public void testDeletesCorrectIndexes() throws Exception {
-
- Integer numDays = 5;
-
- Date createStartDate = new Date();
-
- createStartDate.setTime(yesterday.getTime() - TimeUnit.DAYS.toMillis(numDays - 1));
-
- ElasticsearchDataPruner pruner = new ElasticsearchDataPruner(yesterday, 30, configuration,client(), "*");
- String indexesToDelete = "sensor_index_" + new SimpleDateFormat("yyyy.MM.dd").format(createStartDate) + ".*";
- Boolean deleted = pruner.deleteIndex(admin(), indexesToDelete);
-
- assertTrue("Index deletion should be acknowledged", deleted);
-
- }
-
- @Test
- public void testHandlesNoIndicesToDelete() throws Exception {
-
- ElasticsearchDataPruner pruner = new ElasticsearchDataPruner(yesterday, 1, configuration, client(), "sensor_index_");
- Long deleteCount = pruner.prune();
- assertEquals("Should have pruned 0 indices", 0L, deleteCount.longValue());
-
-
- }
-
- @Override
- protected Settings nodeSettings(int nodeOrdinal) {
-
- return ImmutableSettings.settingsBuilder()
- .put("node.data", true)
- .put("gateway.type", "none")
- .put("path.data", dataPath.getPath() + "/data")
- .put("path.work", dataPath.getPath() + "/work")
- .put("path.logs", dataPath.getPath() + "/logs")
- .put("cluster.routing.schedule", "50ms")
- .put("node.local", true).build();
-
- }
-
- public Settings indexSettings() {
-
- return ImmutableSettings.settingsBuilder()
- .put("index.store.type", "memory")
- .put("index.store.fs.memory.enabled", "true")
- .put("index.number_of_shards", 1)
- .put("index.number_of_replicas", 0).build();
-
- }
-
-}
\ No newline at end of file
diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java
index 0cc5d28..df485f0 100644
--- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java
+++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerTest.java
@@ -17,28 +17,119 @@
*/
package org.apache.metron.dataloads.bulk;
+import com.carrotsearch.hppc.ObjectObjectHashMap;
import org.apache.commons.collections.IteratorUtils;
import org.apache.metron.TestConstants;
import org.apache.metron.common.configuration.Configuration;
import org.easymock.EasyMock;
-import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.*;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
+import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistRequestBuilder;
+import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
+import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
+import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder;
+import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
-import org.elasticsearch.client.*;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
+import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;
+import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder;
+import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
+import org.elasticsearch.action.admin.indices.flush.*;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.*;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
+import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest;
+import org.elasticsearch.action.admin.indices.recovery.RecoveryRequestBuilder;
+import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder;
+import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
+import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
+import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBuilder;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse;
+import org.elasticsearch.action.admin.indices.shards.IndicesShardStoreRequestBuilder;
+import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest;
+import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequestBuilder;
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
+import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequestBuilder;
+import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusRequest;
+import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusRequestBuilder;
+import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusResponse;
+import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
+import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequestBuilder;
+import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeResponse;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
+import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequest;
+import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerRequestBuilder;
+import org.elasticsearch.action.admin.indices.warmer.delete.DeleteWarmerResponse;
+import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersRequest;
+import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersRequestBuilder;
+import org.elasticsearch.action.admin.indices.warmer.get.GetWarmersResponse;
+import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerRequest;
+import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerRequestBuilder;
+import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.ClusterAdminClient;
+import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.hppc.ObjectObjectOpenHashMap;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.indices.IndexMissingException;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.threadpool.ThreadPool;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.mockito.Matchers;
import org.powermock.api.easymock.PowerMock;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -57,7 +148,6 @@
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.powermock.api.easymock.PowerMock.replayAll;
@@ -73,7 +163,7 @@
private Client indexClient = mock(Client.class);
private AdminClient adminClient = mock(AdminClient.class);
- private IndicesAdminClient indicesAdminClient = mock(FilterClient.IndicesAdmin.class);
+ private IndicesAdminClient indicesAdminClient = new TestIndicesAdminClient();
private DeleteIndexRequestBuilder deleteIndexRequestBuilder = mock(DeleteIndexRequestBuilder.class);
private DeleteIndexRequest deleteIndexRequest = mock(DeleteIndexRequest.class);
private ActionFuture<DeleteIndexResponse> deleteIndexAction = mock(ActionFuture.class);
@@ -98,8 +188,6 @@
when(indexClient.admin()).thenReturn(adminClient);
when(adminClient.indices()).thenReturn(indicesAdminClient);
- when(indicesAdminClient.prepareDelete(Matchers.<String>anyVararg())).thenReturn(deleteIndexRequestBuilder);
- when(indicesAdminClient.delete((DeleteIndexRequest) any())).thenReturn(deleteIndexAction);
when(deleteIndexRequestBuilder.request()).thenReturn(deleteIndexRequest);
when(deleteIndexAction.actionGet()).thenReturn(deleteIndexResponse);
@@ -116,12 +204,13 @@
}
- @Test(expected = IndexMissingException.class)
+ @Test(expected = IndexNotFoundException.class)
public void testWillThrowOnMissingIndex() throws Exception {
- when(indicesAdminClient.delete((DeleteIndexRequest) any())).thenThrow(new IndexMissingException(new Index("Test Exception")));
+ ((TestIndicesAdminClient)indicesAdminClient).throwMissingIndex = true;
ElasticsearchDataPruner pruner = new ElasticsearchDataPruner(testDate, 30, configuration, indexClient,"*");
pruner.deleteIndex(adminClient, "baz");
+ ((TestIndicesAdminClient)indicesAdminClient).throwMissingIndex = false;
}
@@ -133,7 +222,7 @@
ClusterStateRequestBuilder clusterStateRequestBuilder = mock(ClusterStateRequestBuilder.class);
ClusterStateResponse clusterStateResponse = mock(ClusterStateResponse.class);
ClusterState clusterState = mock(ClusterState.class);
- ObjectObjectOpenHashMap<String, IndexMetaData> clusterIndexes = new ObjectObjectOpenHashMap();
+ ObjectObjectHashMap<String, IndexMetaData> clusterIndexes = new ObjectObjectHashMap();
MetaData clusterMetadata = mock(MetaData.class);
when(adminClient.cluster()).thenReturn(clusterAdminClient);
when(clusterAdminClient.prepareState()).thenReturn(clusterStateRequestBuilder);
@@ -172,7 +261,7 @@
@Test
public void testFilter() throws Exception {
- ObjectObjectOpenHashMap<String, IndexMetaData> indexNames = new ObjectObjectOpenHashMap();
+ ObjectObjectHashMap<String, IndexMetaData> indexNames = new ObjectObjectHashMap();
SimpleDateFormat dateChecker = new SimpleDateFormat("yyyyMMdd");
int numDays = 5;
String[] expectedIndices = new String[24];
@@ -208,4 +297,559 @@
}
+ class TestIndicesAdminClient implements IndicesAdminClient {
+
+ public boolean throwMissingIndex = false;
+
+ @Override
+ public ActionFuture<DeleteIndexResponse> delete(DeleteIndexRequest request) {
+
+ if(throwMissingIndex){
+
+ throw new IndexNotFoundException("TEST EXCEPTION!");
+
+ }
+
+ return deleteIndexAction;
+
+ }
+
+
+ @Override
+ public ActionFuture<IndicesExistsResponse> exists(IndicesExistsRequest request) {
+ return null;
+ }
+
+ @Override
+ public void exists(IndicesExistsRequest request, ActionListener<IndicesExistsResponse> listener) {
+
+ }
+
+ @Override
+ public IndicesExistsRequestBuilder prepareExists(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<TypesExistsResponse> typesExists(TypesExistsRequest request) {
+ return null;
+ }
+
+ @Override
+ public void typesExists(TypesExistsRequest request, ActionListener<TypesExistsResponse> listener) {
+
+ }
+
+ @Override
+ public TypesExistsRequestBuilder prepareTypesExists(String... index) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<IndicesStatsResponse> stats(IndicesStatsRequest request) {
+ return null;
+ }
+
+ @Override
+ public void stats(IndicesStatsRequest request, ActionListener<IndicesStatsResponse> listener) {
+
+ }
+
+ @Override
+ public IndicesStatsRequestBuilder prepareStats(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<RecoveryResponse> recoveries(RecoveryRequest request) {
+ return null;
+ }
+
+ @Override
+ public void recoveries(RecoveryRequest request, ActionListener<RecoveryResponse> listener) {
+
+ }
+
+ @Override
+ public RecoveryRequestBuilder prepareRecoveries(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<IndicesSegmentResponse> segments(IndicesSegmentsRequest request) {
+ return null;
+ }
+
+ @Override
+ public void segments(IndicesSegmentsRequest request, ActionListener<IndicesSegmentResponse> listener) {
+
+ }
+
+ @Override
+ public IndicesSegmentsRequestBuilder prepareSegments(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<IndicesShardStoresResponse> shardStores(IndicesShardStoresRequest request) {
+ return null;
+ }
+
+ @Override
+ public void shardStores(IndicesShardStoresRequest request, ActionListener<IndicesShardStoresResponse> listener) {
+
+ }
+
+ @Override
+ public IndicesShardStoreRequestBuilder prepareShardStores(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<CreateIndexResponse> create(CreateIndexRequest request) {
+ return null;
+ }
+
+ @Override
+ public void create(CreateIndexRequest request, ActionListener<CreateIndexResponse> listener) {
+
+ }
+
+ @Override
+ public CreateIndexRequestBuilder prepareCreate(String index) {
+ return null;
+ }
+
+
+ @Override
+ public void delete(DeleteIndexRequest request, ActionListener<DeleteIndexResponse> listener) {
+
+ }
+
+ @Override
+ public DeleteIndexRequestBuilder prepareDelete(String... indices) {
+ return deleteIndexRequestBuilder;
+ }
+
+ @Override
+ public ActionFuture<CloseIndexResponse> close(CloseIndexRequest request) {
+ return null;
+ }
+
+ @Override
+ public void close(CloseIndexRequest request, ActionListener<CloseIndexResponse> listener) {
+
+ }
+
+ @Override
+ public CloseIndexRequestBuilder prepareClose(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<OpenIndexResponse> open(OpenIndexRequest request) {
+ return null;
+ }
+
+ @Override
+ public void open(OpenIndexRequest request, ActionListener<OpenIndexResponse> listener) {
+
+ }
+
+ @Override
+ public OpenIndexRequestBuilder prepareOpen(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<RefreshResponse> refresh(RefreshRequest request) {
+ return null;
+ }
+
+ @Override
+ public void refresh(RefreshRequest request, ActionListener<RefreshResponse> listener) {
+
+ }
+
+ @Override
+ public RefreshRequestBuilder prepareRefresh(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<FlushResponse> flush(FlushRequest request) {
+ return null;
+ }
+
+ @Override
+ public void flush(FlushRequest request, ActionListener<FlushResponse> listener) {
+
+ }
+
+ @Override
+ public FlushRequestBuilder prepareFlush(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<SyncedFlushResponse> syncedFlush(SyncedFlushRequest request) {
+ return null;
+ }
+
+ @Override
+ public void syncedFlush(SyncedFlushRequest request, ActionListener<SyncedFlushResponse> listener) {
+
+ }
+
+ @Override
+ public SyncedFlushRequestBuilder prepareSyncedFlush(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<ForceMergeResponse> forceMerge(ForceMergeRequest request) {
+ return null;
+ }
+
+ @Override
+ public void forceMerge(ForceMergeRequest request, ActionListener<ForceMergeResponse> listener) {
+
+ }
+
+ @Override
+ public ForceMergeRequestBuilder prepareForceMerge(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<UpgradeResponse> upgrade(UpgradeRequest request) {
+ return null;
+ }
+
+ @Override
+ public void upgrade(UpgradeRequest request, ActionListener<UpgradeResponse> listener) {
+
+ }
+
+ @Override
+ public UpgradeStatusRequestBuilder prepareUpgradeStatus(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<UpgradeStatusResponse> upgradeStatus(UpgradeStatusRequest request) {
+ return null;
+ }
+
+ @Override
+ public void upgradeStatus(UpgradeStatusRequest request, ActionListener<UpgradeStatusResponse> listener) {
+
+ }
+
+ @Override
+ public UpgradeRequestBuilder prepareUpgrade(String... indices) {
+ return null;
+ }
+
+ @Override
+ public void getMappings(GetMappingsRequest request, ActionListener<GetMappingsResponse> listener) {
+
+ }
+
+ @Override
+ public ActionFuture<GetMappingsResponse> getMappings(GetMappingsRequest request) {
+ return null;
+ }
+
+ @Override
+ public GetMappingsRequestBuilder prepareGetMappings(String... indices) {
+ return null;
+ }
+
+ @Override
+ public void getFieldMappings(GetFieldMappingsRequest request, ActionListener<GetFieldMappingsResponse> listener) {
+
+ }
+
+ @Override
+ public GetFieldMappingsRequestBuilder prepareGetFieldMappings(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<GetFieldMappingsResponse> getFieldMappings(GetFieldMappingsRequest request) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<PutMappingResponse> putMapping(PutMappingRequest request) {
+ return null;
+ }
+
+ @Override
+ public void putMapping(PutMappingRequest request, ActionListener<PutMappingResponse> listener) {
+
+ }
+
+ @Override
+ public PutMappingRequestBuilder preparePutMapping(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<IndicesAliasesResponse> aliases(IndicesAliasesRequest request) {
+ return null;
+ }
+
+ @Override
+ public void aliases(IndicesAliasesRequest request, ActionListener<IndicesAliasesResponse> listener) {
+
+ }
+
+ @Override
+ public IndicesAliasesRequestBuilder prepareAliases() {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<GetAliasesResponse> getAliases(GetAliasesRequest request) {
+ return null;
+ }
+
+ @Override
+ public void getAliases(GetAliasesRequest request, ActionListener<GetAliasesResponse> listener) {
+
+ }
+
+ @Override
+ public GetAliasesRequestBuilder prepareGetAliases(String... aliases) {
+ return null;
+ }
+
+ @Override
+ public AliasesExistRequestBuilder prepareAliasesExist(String... aliases) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<AliasesExistResponse> aliasesExist(GetAliasesRequest request) {
+ return null;
+ }
+
+ @Override
+ public void aliasesExist(GetAliasesRequest request, ActionListener<AliasesExistResponse> listener) {
+
+ }
+
+ @Override
+ public ActionFuture<GetIndexResponse> getIndex(GetIndexRequest request) {
+ return null;
+ }
+
+ @Override
+ public void getIndex(GetIndexRequest request, ActionListener<GetIndexResponse> listener) {
+
+ }
+
+ @Override
+ public GetIndexRequestBuilder prepareGetIndex() {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<ClearIndicesCacheResponse> clearCache(ClearIndicesCacheRequest request) {
+ return null;
+ }
+
+ @Override
+ public void clearCache(ClearIndicesCacheRequest request, ActionListener<ClearIndicesCacheResponse> listener) {
+
+ }
+
+ @Override
+ public ClearIndicesCacheRequestBuilder prepareClearCache(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<UpdateSettingsResponse> updateSettings(UpdateSettingsRequest request) {
+ return null;
+ }
+
+ @Override
+ public void updateSettings(UpdateSettingsRequest request, ActionListener<UpdateSettingsResponse> listener) {
+
+ }
+
+ @Override
+ public UpdateSettingsRequestBuilder prepareUpdateSettings(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<AnalyzeResponse> analyze(AnalyzeRequest request) {
+ return null;
+ }
+
+ @Override
+ public void analyze(AnalyzeRequest request, ActionListener<AnalyzeResponse> listener) {
+
+ }
+
+ @Override
+ public AnalyzeRequestBuilder prepareAnalyze(@Nullable String index, String text) {
+ return null;
+ }
+
+ @Override
+ public AnalyzeRequestBuilder prepareAnalyze(String text) {
+ return null;
+ }
+
+ @Override
+ public AnalyzeRequestBuilder prepareAnalyze() {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<PutIndexTemplateResponse> putTemplate(PutIndexTemplateRequest request) {
+ return null;
+ }
+
+ @Override
+ public void putTemplate(PutIndexTemplateRequest request, ActionListener<PutIndexTemplateResponse> listener) {
+
+ }
+
+ @Override
+ public PutIndexTemplateRequestBuilder preparePutTemplate(String name) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<DeleteIndexTemplateResponse> deleteTemplate(DeleteIndexTemplateRequest request) {
+ return null;
+ }
+
+ @Override
+ public void deleteTemplate(DeleteIndexTemplateRequest request, ActionListener<DeleteIndexTemplateResponse> listener) {
+
+ }
+
+ @Override
+ public DeleteIndexTemplateRequestBuilder prepareDeleteTemplate(String name) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<GetIndexTemplatesResponse> getTemplates(GetIndexTemplatesRequest request) {
+ return null;
+ }
+
+ @Override
+ public void getTemplates(GetIndexTemplatesRequest request, ActionListener<GetIndexTemplatesResponse> listener) {
+
+ }
+
+ @Override
+ public GetIndexTemplatesRequestBuilder prepareGetTemplates(String... name) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<ValidateQueryResponse> validateQuery(ValidateQueryRequest request) {
+ return null;
+ }
+
+ @Override
+ public void validateQuery(ValidateQueryRequest request, ActionListener<ValidateQueryResponse> listener) {
+
+ }
+
+ @Override
+ public ValidateQueryRequestBuilder prepareValidateQuery(String... indices) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<PutWarmerResponse> putWarmer(PutWarmerRequest request) {
+ return null;
+ }
+
+ @Override
+ public void putWarmer(PutWarmerRequest request, ActionListener<PutWarmerResponse> listener) {
+
+ }
+
+ @Override
+ public PutWarmerRequestBuilder preparePutWarmer(String name) {
+ return null;
+ }
+
+ @Override
+ public ActionFuture<DeleteWarmerResponse> deleteWarmer(DeleteWarmerRequest request) {
+ return null;
+ }
+
+ @Override
+ public void deleteWarmer(DeleteWarmerRequest request, ActionListener<DeleteWarmerResponse> listener) {
+
+ }
+
+ @Override
+ public DeleteWarmerRequestBuilder prepareDeleteWarmer() {
+ return null;
+ }
+
+ @Override
+ public void getWarmers(GetWarmersRequest request, ActionListener<GetWarmersResponse> listener) {
+
+ }
+
+ @Override
+ public ActionFuture<GetWarmersResponse> getWarmers(GetWarmersRequest request) {
+ return null;
+ }
+
+ @Override
+ public GetWarmersRequestBuilder prepareGetWarmers(String... indices) {
+ return null;
+ }
+
+ @Override
+ public void getSettings(GetSettingsRequest request, ActionListener<GetSettingsResponse> listener) {
+
+ }
+
+ @Override
+ public ActionFuture<GetSettingsResponse> getSettings(GetSettingsRequest request) {
+ return null;
+ }
+
+ @Override
+ public GetSettingsRequestBuilder prepareGetSettings(String... indices) {
+ return null;
+ }
+
+ @Override
+ public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
+ return null;
+ }
+
+ @Override
+ public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+
+ }
+
+ @Override
+ public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
+ return null;
+ }
+
+ @Override
+ public ThreadPool threadPool() {
+ return null;
+ }
+ }
+
}
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 90b091a..fb6f929 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -33,13 +33,13 @@
</dependency>
<dependency>
<groupId>org.apache.metron</groupId>
- <artifactId>metron-enrichment</artifactId>
+ <artifactId>elasticsearch-shaded</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>${global_elasticsearch_version}</version>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-enrichment</artifactId>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
@@ -106,6 +106,21 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.10.5.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.2.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.2.3</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.metron</groupId>
<artifactId>metron-integration-test</artifactId>
<version>${project.parent.version}</version>
@@ -117,7 +132,6 @@
<version>${global_mockito_version}</version>
<scope>test</scope>
</dependency>
-
</dependencies>
<reporting>
<plugins>
@@ -206,6 +220,16 @@
<goal>shade</goal>
</goals>
<configuration>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.metron.guava.metron-elasticsearch</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson.core</pattern>
+ <shadedPattern>com.fasterxml.jackson.core.metron.elasticsearch</shadedPattern>
+ </relocation>
+ </relocations>
<artifactSet>
<excludes>
<exclude>storm:storm-core:*</exclude>
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchFieldNameConverter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchFieldNameConverter.java
new file mode 100644
index 0000000..57e07ea
--- /dev/null
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchFieldNameConverter.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.elasticsearch.writer;
+
+import org.apache.metron.common.interfaces.FieldNameConverter;
+import java.io.Serializable;
+
+public class ElasticsearchFieldNameConverter implements FieldNameConverter, Serializable {
+
+ private static final long serialVersionUID = -3126840090749760299L;
+
+ @Override
+ public String convert(String originalField) {
+ return originalField.replace(".",":");
+ }
+
+}
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
index 4aa5995..f06850b 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java
@@ -21,17 +21,20 @@
import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
import org.apache.metron.common.interfaces.BulkMessageWriter;
+import org.apache.metron.common.interfaces.FieldNameConverter;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Serializable;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
@@ -44,6 +47,7 @@
private SimpleDateFormat dateFormat;
private static final Logger LOG = LoggerFactory
.getLogger(ElasticsearchWriter.class);
+ private FieldNameConverter fieldNameConverter = new ElasticsearchFieldNameConverter();
public ElasticsearchWriter withOptionalSettings(Map<String, String> optionalSettings) {
this.optionalSettings = optionalSettings;
@@ -53,14 +57,32 @@
@Override
public void init(Map stormConf, EnrichmentConfigurations configurations) {
Map<String, Object> globalConfiguration = configurations.getGlobalConfig();
- ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder();
- builder.put("cluster.name", globalConfiguration.get("es.clustername"));
- builder.put("client.transport.ping_timeout","500s");
+
+ Settings.Builder settingsBuilder = Settings.settingsBuilder();
+ settingsBuilder.put("cluster.name", globalConfiguration.get("es.clustername"));
+ settingsBuilder.put("client.transport.ping_timeout","500s");
+
if (optionalSettings != null) {
- builder.put(optionalSettings);
+
+ settingsBuilder.put(optionalSettings);
+
}
- client = new TransportClient(builder.build())
- .addTransportAddress(new InetSocketTransportAddress(globalConfiguration.get("es.ip").toString(), Integer.parseInt(globalConfiguration.get("es.port").toString())));
+
+ Settings settings = settingsBuilder.build();
+
+ try{
+
+ client = TransportClient.builder().settings(settings).build()
+ .addTransportAddress(
+ new InetSocketTransportAddress(InetAddress.getByName(globalConfiguration.get("es.ip").toString()), Integer.parseInt(globalConfiguration.get("es.port").toString()) )
+ );
+
+
+ } catch (UnknownHostException exception){
+
+ throw new RuntimeException(exception);
+ }
+
dateFormat = new SimpleDateFormat((String) globalConfiguration.get("es.date.format"));
}
@@ -70,25 +92,61 @@
SensorEnrichmentConfig sensorEnrichmentConfig = configurations.getSensorEnrichmentConfig(sensorType);
String indexPostfix = dateFormat.format(new Date());
BulkRequestBuilder bulkRequest = client.prepareBulk();
+
for(JSONObject message: messages) {
+
String indexName = sensorType;
+
if (sensorEnrichmentConfig != null) {
indexName = sensorEnrichmentConfig.getIndex();
}
- IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName + "_index_" + indexPostfix,
- sensorType + "_doc");
- indexRequestBuilder.setSource(message.toJSONString());
+ indexName = indexName + "_index_" + indexPostfix;
+
+ JSONObject esDoc = new JSONObject();
+ for(Object k : message.keySet()){
+
+ deDot(k.toString(),message,esDoc);
+
+ }
+
+ IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName,
+ sensorType + "_doc");
+ indexRequestBuilder.setSource(esDoc.toJSONString()).setTimestamp(esDoc.get("timestamp").toString());
bulkRequest.add(indexRequestBuilder);
+
}
+
BulkResponse resp = bulkRequest.execute().actionGet();
+
if (resp.hasFailures()) {
+
throw new Exception(resp.buildFailureMessage());
+
}
+
}
@Override
public void close() throws Exception {
client.close();
}
+
+ //JSONObject doesn't expose map generics
+ @SuppressWarnings("unchecked")
+ private void deDot(String field, JSONObject origMessage, JSONObject message){
+
+ if(field.contains(".")){
+
+ if(LOG.isDebugEnabled()){
+ LOG.debug("Dotted field: " + field);
+ }
+
+ }
+ String newkey = fieldNameConverter.convert(field);
+ message.put(newkey,origMessage.get(field));
+
+ }
+
}
+
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
index f7d96a4..e0bbb89 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchEnrichmentIntegrationTest.java
@@ -17,6 +17,9 @@
*/
package org.apache.metron.elasticsearch.integration;
+import org.apache.metron.TestConstants;
+import org.apache.metron.common.interfaces.FieldNameConverter;
+import org.apache.metron.elasticsearch.writer.ElasticsearchFieldNameConverter;
import org.apache.metron.integration.EnrichmentIntegrationTest;
import org.apache.metron.integration.ComponentRunner;
import org.apache.metron.integration.InMemoryComponent;
@@ -37,6 +40,12 @@
private String indexDir = "target/elasticsearch";
private String dateFormat = "yyyy.MM.dd.HH";
private String index = "yaf_index_" + new SimpleDateFormat(dateFormat).format(new Date());
+ private FieldNameConverter fieldNameConverter = new ElasticsearchFieldNameConverter();
+
+ @Override
+ public FieldNameConverter getFieldNameConverter() {
+ return fieldNameConverter;
+ }
@Override
public InMemoryComponent getSearchComponent(final Properties topologyProperties) {
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
index 61d50f1..d67361b 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
@@ -24,16 +24,13 @@
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
@@ -103,30 +100,30 @@
} catch (IOException e) {
throw new UnableToStartException("Unable to clean log or data directories", e);
}
- ImmutableSettings.Builder immutableSettings = ImmutableSettings.settingsBuilder()
+
+ Settings.Builder settingsBuilder = Settings.settingsBuilder()
.put("node.http.enabled", true)
.put("http.port", httpPort)
- .put("cluster.name", "metron")
.put("path.logs",logDir.getAbsolutePath())
.put("path.data",dataDir.getAbsolutePath())
- .put("gateway.type", "none")
- .put("index.store.type", "memory")
+ .put("path.home", indexDir.getAbsoluteFile())
.put("index.number_of_shards", 1)
.put("node.mode", "network")
.put("index.number_of_replicas", 1);
+
if(extraElasticSearchSettings != null) {
- immutableSettings = immutableSettings.put(extraElasticSearchSettings);
+
+ settingsBuilder = settingsBuilder.put(extraElasticSearchSettings);
+
}
- Settings settings = immutableSettings.build();
- node = NodeBuilder.nodeBuilder().settings(settings).node();
+
+ node = NodeBuilder.nodeBuilder().settings(settingsBuilder).clusterName("metron").node();
node.start();
- settings = ImmutableSettings.settingsBuilder()
- .put("cluster.name", "metron").build();
- client = new TransportClient(settings)
- .addTransportAddress(new InetSocketTransportAddress("localhost",
- 9300));
+
+ client = node.client();
waitForCluster(client, ClusterHealthStatus.YELLOW, new TimeValue(60000));
+
}
public static void waitForCluster(ElasticsearchClient client, ClusterHealthStatus status, TimeValue timeout) throws UnableToStartException {
@@ -179,7 +176,7 @@
}
public void stop() {
- node.stop();
+ node.close();
node = null;
client = null;
}
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchFieldNameConverterTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchFieldNameConverterTest.java
new file mode 100644
index 0000000..3e52581
--- /dev/null
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/writer/ElasticsearchFieldNameConverterTest.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.elasticsearch.writer;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class ElasticsearchFieldNameConverterTest {
+
+ @Test
+ public void convert() throws Exception {
+ assertEquals("testfield:with:colons",new ElasticsearchFieldNameConverter().convert("testfield.with.colons"));
+ }
+
+}
\ No newline at end of file
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
index 111dbff..fb3febe 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/geo/GeoAdapter.java
@@ -70,7 +70,7 @@
enriched.put("latitude", resultSet.getString("latitude"));
enriched.put("longitude", resultSet.getString("longitude"));
enriched.put("dmaCode", resultSet.getString("dmaCode"));
- enriched.put("location_point", enriched.get("longitude") + "," + enriched.get("latitude"));
+ enriched.put("location_point", enriched.get("latitude") + "," + enriched.get("longitude"));
}
resultSet.close();
} catch (Exception e) {
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java
index ac5adde..061ff3f 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/geo/GeoAdapterTest.java
@@ -47,7 +47,7 @@
* "latitude":"test latitude",
* "longitude":"test longitude",
* "dmaCode":"test dma",
- * "location_point":"test longitude,test latitude"
+ * "location_point":"test latitude,test longitude"
* }
*/
@Multiline
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
index 818dae0..521ffdf 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
@@ -26,6 +26,7 @@
import org.apache.metron.common.Constants;
import org.apache.metron.TestConstants;
import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.interfaces.FieldNameConverter;
import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.hbase.TableProvider;
import org.apache.metron.enrichment.converter.EnrichmentKey;
@@ -67,13 +68,12 @@
private static final Map<String, String> PLAYFUL_ENRICHMENT = new HashMap<String, String>() {{
put("orientation", "north");
}};
- private String fluxPath = "../metron-enrichment/src/main/flux/enrichment/test.yaml";
protected String testSensorType = "test";
protected String hdfsDir = "target/enrichmentIntegrationTest/hdfs";
- private String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
+ protected String fluxPath = "../metron-enrichment/src/main/flux/enrichment/test.yaml";
+ protected String sampleParsedPath = TestConstants.SAMPLE_DATA_PARSED_PATH + "TestExampleParsed";
private String sampleIndexedPath = TestConstants.SAMPLE_DATA_INDEXED_PATH + "TestIndexed";
-
public static class Provider implements TableProvider, Serializable {
MockHTable.Provider provider = new MockHTable.Provider();
@Override
@@ -218,14 +218,15 @@
List<Map<String, Object>> docs = runner.process(getProcessor(inputMessages));
Assert.assertEquals(inputMessages.size(), docs.size());
List<Map<String, Object>> cleanedDocs = cleanDocs(docs);
- validateAll(cleanedDocs);
+ validateAll(cleanedDocs, getFieldNameConverter());
List<Map<String, Object>> docsFromDisk = readDocsFromDisk(hdfsDir);
Assert.assertEquals(docsFromDisk.size(), docs.size()) ;
Assert.assertEquals(new File(hdfsDir).list().length, 1);
Assert.assertEquals(new File(hdfsDir).list()[0], testSensorType);
- validateAll(docsFromDisk);
+ //we want the identity transformation when dealing with docs on disk.
+ validateAll(docsFromDisk, fieldName -> fieldName);
}
finally {
cleanHdfsDir(hdfsDir);
@@ -245,17 +246,18 @@
return cleanedDocs;
}
- public static void validateAll(List<Map<String, Object>> docs) {
+ public static void validateAll(List<Map<String, Object>> docs, FieldNameConverter fnc) {
+
for (Map<String, Object> doc : docs) {
- baseValidation(doc);
- hostEnrichmentValidation(doc);
- geoEnrichmentValidation(doc);
- threatIntelValidation(doc);
- simpleEnrichmentValidation(doc);
+ baseValidation(doc, fnc);
+ hostEnrichmentValidation(doc, fnc);
+ geoEnrichmentValidation(doc, fnc);
+ threatIntelValidation(doc, fnc);
+ simpleEnrichmentValidation(doc, fnc);
}
}
- public static void baseValidation(Map<String, Object> jsonDoc) {
+ public static void baseValidation(Map<String, Object> jsonDoc, FieldNameConverter fnc) {
assertEnrichmentsExists("threatintels.", setOf("hbaseThreatIntel"), jsonDoc.keySet());
assertEnrichmentsExists("enrichments.", setOf("geo", "host", "hbaseEnrichment" ), jsonDoc.keySet());
for(Map.Entry<String, Object> kv : jsonDoc.entrySet()) {
@@ -270,49 +272,55 @@
private static class EvaluationPayload {
Map<String, Object> indexedDoc;
String key;
- public EvaluationPayload(Map<String, Object> indexedDoc, String key) {
+ FieldNameConverter fnc;
+ public EvaluationPayload(Map<String, Object> indexedDoc, String key, FieldNameConverter fnc) {
this.indexedDoc = indexedDoc;
this.key = key;
+ this.fnc = fnc;
}
}
private static enum HostEnrichments implements Predicate<EvaluationPayload>{
+
LOCAL_LOCATION(new Predicate<EvaluationPayload>() {
@Override
public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
- return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.local").equals("YES");
+
+ return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.local"),"").equals("YES");
+
}
})
+
,UNKNOWN_LOCATION(new Predicate<EvaluationPayload>() {
@Override
public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
- return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.local").equals("UNKNOWN");
+ return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.local"),"").equals("UNKNOWN");
}
})
,IMPORTANT(new Predicate<EvaluationPayload>() {
@Override
public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
- return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.asset_value").equals("important");
+ return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.asset_value"),"").equals("important");
}
})
,PRINTER_TYPE(new Predicate<EvaluationPayload>() {
@Override
public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
- return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.type").equals("printer");
+ return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.type"),"").equals("printer");
}
})
,WEBSERVER_TYPE(new Predicate<EvaluationPayload>() {
@Override
public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
- return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.type").equals("webserver");
+ return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.type"),"").equals("webserver");
}
})
,UNKNOWN_TYPE(new Predicate<EvaluationPayload>() {
@Override
public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
- return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.type").equals("unknown");
+ return evaluationPayload.indexedDoc.getOrDefault(evaluationPayload.fnc.convert("enrichments.host." + evaluationPayload.key + ".known_info.type"),"").equals("unknown");
}
})
;
@@ -340,46 +348,46 @@
}
}
}
- private static void simpleEnrichmentValidation(Map<String, Object> indexedDoc) {
- if(indexedDoc.get(SRC_IP).equals("10.0.2.3")
- || indexedDoc.get(DST_IP).equals("10.0.2.3")
+ private static void simpleEnrichmentValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
+ if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")
+ || indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")
) {
- Assert.assertTrue(keyPatternExists("enrichments.hbaseEnrichment", indexedDoc));
- if(indexedDoc.get(SRC_IP).equals("10.0.2.3")) {
- Assert.assertEquals(indexedDoc.get("enrichments.hbaseEnrichment." + SRC_IP + "." + PLAYFUL_CLASSIFICATION_TYPE+ ".orientation")
+ Assert.assertTrue(keyPatternExists(fnc.convert("enrichments.hbaseEnrichment"), indexedDoc));
+ if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")) {
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.hbaseEnrichment." + SRC_IP + "." + PLAYFUL_CLASSIFICATION_TYPE+ ".orientation"))
, PLAYFUL_ENRICHMENT.get("orientation")
);
}
- else if(indexedDoc.get(DST_IP).equals("10.0.2.3")) {
- Assert.assertEquals( indexedDoc.get("enrichments.hbaseEnrichment." + DST_IP + "." + PLAYFUL_CLASSIFICATION_TYPE + ".orientation")
+ else if(indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")) {
+ Assert.assertEquals( indexedDoc.get(fnc.convert("enrichments.hbaseEnrichment." + DST_IP + "." + PLAYFUL_CLASSIFICATION_TYPE + ".orientation"))
, PLAYFUL_ENRICHMENT.get("orientation")
);
}
}
}
- private static void threatIntelValidation(Map<String, Object> indexedDoc) {
- if(indexedDoc.get(SRC_IP).equals("10.0.2.3")
- || indexedDoc.get(DST_IP).equals("10.0.2.3")
+ private static void threatIntelValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
+ if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")
+ || indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")
) {
//if we have any threat intel messages, we want to tag is_alert to true
- Assert.assertTrue(keyPatternExists("threatintels.", indexedDoc));
- Assert.assertTrue(indexedDoc.containsKey("threat.triage.level"));
- Assert.assertEquals(indexedDoc.get("is_alert"), "true");
- Assert.assertEquals((double)indexedDoc.get("threat.triage.level"), 10d, 1e-7);
+ Assert.assertTrue(keyPatternExists(fnc.convert("threatintels."), indexedDoc));
+ Assert.assertTrue(indexedDoc.containsKey(fnc.convert("threat.triage.level")));
+ Assert.assertEquals(indexedDoc.getOrDefault(fnc.convert("is_alert"),""), "true");
+ Assert.assertEquals((double)indexedDoc.get(fnc.convert("threat.triage.level")), 10d, 1e-7);
}
else {
//For YAF this is the case, but if we do snort later on, this will be invalid.
- Assert.assertNull(indexedDoc.get("is_alert"));
- Assert.assertFalse(keyPatternExists("threatintels.", indexedDoc));
+ Assert.assertNull(indexedDoc.get(fnc.convert("is_alert")));
+ Assert.assertFalse(keyPatternExists(fnc.convert("threatintels."), indexedDoc));
}
//ip threat intels
- if(keyPatternExists("threatintels.hbaseThreatIntel.", indexedDoc)) {
- if(indexedDoc.get(SRC_IP).equals("10.0.2.3")) {
- Assert.assertEquals(indexedDoc.get("threatintels.hbaseThreatIntel." + SRC_IP + "." + MALICIOUS_IP_TYPE), "alert");
+ if(keyPatternExists(fnc.convert("threatintels.hbaseThreatIntel."), indexedDoc)) {
+ if(indexedDoc.getOrDefault(fnc.convert(SRC_IP),"").equals("10.0.2.3")) {
+ Assert.assertEquals(indexedDoc.get(fnc.convert("threatintels.hbaseThreatIntel." + SRC_IP + "." + MALICIOUS_IP_TYPE)), "alert");
}
- else if(indexedDoc.get(DST_IP).equals("10.0.2.3")) {
- Assert.assertEquals(indexedDoc.get("threatintels.hbaseThreatIntel." + DST_IP + "." + MALICIOUS_IP_TYPE), "alert");
+ else if(indexedDoc.getOrDefault(fnc.convert(DST_IP),"").equals("10.0.2.3")) {
+ Assert.assertEquals(indexedDoc.get(fnc.convert("threatintels.hbaseThreatIntel." + DST_IP + "." + MALICIOUS_IP_TYPE)), "alert");
}
else {
Assert.fail("There was a threat intels that I did not expect: " + indexedDoc);
@@ -388,43 +396,43 @@
}
- private static void geoEnrichmentValidation(Map<String, Object> indexedDoc) {
+ private static void geoEnrichmentValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
//should have geo enrichment on every message due to mock geo adapter
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".location_point"), MockGeoAdapter.DEFAULT_LOCATION_POINT);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP +".location_point"), MockGeoAdapter.DEFAULT_LOCATION_POINT);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".longitude"), MockGeoAdapter.DEFAULT_LONGITUDE);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".longitude"), MockGeoAdapter.DEFAULT_LONGITUDE);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".city"), MockGeoAdapter.DEFAULT_CITY);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".city"), MockGeoAdapter.DEFAULT_CITY);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".latitude"), MockGeoAdapter.DEFAULT_LATITUDE);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".latitude"), MockGeoAdapter.DEFAULT_LATITUDE);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".country"), MockGeoAdapter.DEFAULT_COUNTRY);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".country"), MockGeoAdapter.DEFAULT_COUNTRY);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".dmaCode"), MockGeoAdapter.DEFAULT_DMACODE);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".dmaCode"), MockGeoAdapter.DEFAULT_DMACODE);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + DST_IP + ".postalCode"), MockGeoAdapter.DEFAULT_POSTAL_CODE);
- Assert.assertEquals(indexedDoc.get("enrichments.geo." + SRC_IP + ".postalCode"), MockGeoAdapter.DEFAULT_POSTAL_CODE);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".location_point")), MockGeoAdapter.DEFAULT_LOCATION_POINT);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP +".location_point")), MockGeoAdapter.DEFAULT_LOCATION_POINT);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".longitude")), MockGeoAdapter.DEFAULT_LONGITUDE);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".longitude")), MockGeoAdapter.DEFAULT_LONGITUDE);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".city")), MockGeoAdapter.DEFAULT_CITY);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".city")), MockGeoAdapter.DEFAULT_CITY);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".latitude")), MockGeoAdapter.DEFAULT_LATITUDE);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".latitude")), MockGeoAdapter.DEFAULT_LATITUDE);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".country")), MockGeoAdapter.DEFAULT_COUNTRY);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".country")), MockGeoAdapter.DEFAULT_COUNTRY);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".dmaCode")), MockGeoAdapter.DEFAULT_DMACODE);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".dmaCode")), MockGeoAdapter.DEFAULT_DMACODE);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + DST_IP + ".postalCode")), MockGeoAdapter.DEFAULT_POSTAL_CODE);
+ Assert.assertEquals(indexedDoc.get(fnc.convert("enrichments.geo." + SRC_IP + ".postalCode")), MockGeoAdapter.DEFAULT_POSTAL_CODE);
}
- private static void hostEnrichmentValidation(Map<String, Object> indexedDoc) {
+ private static void hostEnrichmentValidation(Map<String, Object> indexedDoc, FieldNameConverter fnc) {
boolean enriched = false;
//important local printers
{
Set<String> ips = setOf("10.0.2.15", "10.60.10.254");
- if (ips.contains(indexedDoc.get(SRC_IP))) {
+ if (ips.contains(indexedDoc.get(fnc.convert(SRC_IP)))) {
//this is a local, important, printer
Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
,HostEnrichments.IMPORTANT
,HostEnrichments.PRINTER_TYPE
- ).apply(new EvaluationPayload(indexedDoc, SRC_IP))
+ ).apply(new EvaluationPayload(indexedDoc, SRC_IP, fnc))
);
enriched = true;
}
- if (ips.contains(indexedDoc.get(DST_IP))) {
+ if (ips.contains(indexedDoc.get(fnc.convert(DST_IP)))) {
Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
,HostEnrichments.IMPORTANT
,HostEnrichments.PRINTER_TYPE
- ).apply(new EvaluationPayload(indexedDoc, DST_IP))
+ ).apply(new EvaluationPayload(indexedDoc, DST_IP, fnc))
);
enriched = true;
}
@@ -432,20 +440,20 @@
//important local webservers
{
Set<String> ips = setOf("10.1.128.236");
- if (ips.contains(indexedDoc.get(SRC_IP))) {
+ if (ips.contains(indexedDoc.get(fnc.convert(SRC_IP)))) {
//this is a local, important, printer
Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
,HostEnrichments.IMPORTANT
,HostEnrichments.WEBSERVER_TYPE
- ).apply(new EvaluationPayload(indexedDoc, SRC_IP))
+ ).apply(new EvaluationPayload(indexedDoc, SRC_IP, fnc))
);
enriched = true;
}
- if (ips.contains(indexedDoc.get(DST_IP))) {
+ if (ips.contains(indexedDoc.get(fnc.convert(DST_IP)))) {
Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
,HostEnrichments.IMPORTANT
,HostEnrichments.WEBSERVER_TYPE
- ).apply(new EvaluationPayload(indexedDoc, DST_IP))
+ ).apply(new EvaluationPayload(indexedDoc, DST_IP, fnc))
);
enriched = true;
}
@@ -472,6 +480,7 @@
return ret;
}
+ abstract public FieldNameConverter getFieldNameConverter();
abstract public InMemoryComponent getSearchComponent(Properties topologyProperties) throws Exception;
abstract public Processor<List<Map<String, Object>>> getProcessor(List<byte[]> inputMessages);
abstract public void setAdditionalProperties(Properties topologyProperties);
diff --git a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java
index 70791fe..7316112 100644
--- a/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java
+++ b/metron-platform/metron-integration-test/src/main/java/org/apache/metron/integration/mock/MockGeoAdapter.java
@@ -34,7 +34,7 @@
public static final String DEFAULT_LATITUDE = "test latitude";
public static final String DEFAULT_LONGITUDE = "test longitude";
public static final String DEFAULT_DMACODE= "test dmaCode";
- public static final String DEFAULT_LOCATION_POINT= Joiner.on(',').join(DEFAULT_LONGITUDE, DEFAULT_LATITUDE);
+ public static final String DEFAULT_LOCATION_POINT= Joiner.on(',').join(DEFAULT_LATITUDE,DEFAULT_LONGITUDE);
@Override
public void logAccess(CacheKey value) {
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/test/indexed/TestIndexed b/metron-platform/metron-integration-test/src/main/sample/data/test/indexed/TestIndexed
index 0dfbe43..75af86b 100644
--- a/metron-platform/metron-integration-test/src/main/sample/data/test/indexed/TestIndexed
+++ b/metron-platform/metron-integration-test/src/main/sample/data/test/indexed/TestIndexed
@@ -1,10 +1,10 @@
-{"adapter.threatinteladapter.end.ts":"1457102731219","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa001","index.elasticsearchwriter.ts":"1457102731220","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731206","adapter.hostfromjsonlistadapter.begin.ts":"1457102731185","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":44,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731185","threatintelsplitterbolt.splitter.ts":"1457102731207","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731210","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AS","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731220","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.host.dip.known_info.asset_value":"important","source.type":"test","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":10000000,"index.elasticsearchwriter.ts":"1457102731221","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731208","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitter.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988502,"adapter.threatinteladapter.begin.ts":"1457102731219","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731221","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":37299,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988502,"risn":0,"end_time":1453994988502,"is_alert":"true","source.type":"test","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":37299,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":312,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitter.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988504,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988504,"enrichments.host.dip.known_info.asset_value":"important","is_alert":"true","source.type":"test","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731211","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":56303,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988504,"risn":0,"end_time":1453994988504,"is_alert":"true","source.type":"test","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":56303,"rpkt":0,"original_string":"2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":84,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988506,"adapter.threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988506,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988506,"enrichments.host.dip.known_info.asset_value":"important","is_alert":"true","source.type":"test","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fca","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":60,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988508,"adapter.threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"S","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988508,"risn":0,"end_time":1453994988508,"source.type":"test","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"end_time":1453994988512,"source.type":"test","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":148,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"end_time":1453994988512,"source.type":"test","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731225","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.host.dip.known_info.asset_value":"important","source.type":"test","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731226","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":604,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731213","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988562,"adapter.threatinteladapter.begin.ts":"1457102731226","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731226","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988562,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988562,"enrichments.host.dip.known_info.asset_value":"important","source.type":"test","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731219","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":"22efa001","index.elasticsearchwriter.ts":"1457102731220","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731206","adapter.hostfromjsonlistadapter.begin.ts":"1457102731185","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":44,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731185","threatintelsplitterbolt.splitter.ts":"1457102731207","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731210","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AS","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731220","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.host.dip.known_info.asset_value":"important","source.type":"test","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":10000000,"index.elasticsearchwriter.ts":"1457102731221","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731208","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitter.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988502,"adapter.threatinteladapter.begin.ts":"1457102731219","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731221","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":37299,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988502,"risn":0,"end_time":1453994988502,"is_alert":"true","source.type":"test","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":37299,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":312,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitter.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988504,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988504,"enrichments.host.dip.known_info.asset_value":"important","is_alert":"true","source.type":"test","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731211","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":56303,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988504,"risn":0,"end_time":1453994988504,"is_alert":"true","source.type":"test","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":56303,"rpkt":0,"original_string":"2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":84,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988506,"adapter.threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988506,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988506,"enrichments.host.dip.known_info.asset_value":"important","is_alert":"true","source.type":"test","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fca","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":60,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988508,"adapter.threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"S","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988508,"risn":0,"end_time":1453994988508,"source.type":"test","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"end_time":1453994988512,"source.type":"test","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":148,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"end_time":1453994988512,"source.type":"test","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731225","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.host.dip.known_info.asset_value":"important","source.type":"test","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731226","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":604,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731213","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988562,"adapter.threatinteladapter.begin.ts":"1457102731226","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731226","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988562,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988562,"enrichments.host.dip.known_info.asset_value":"important","source.type":"test","rtt":"0.000"}
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/yaf/indexed/YafIndexed b/metron-platform/metron-integration-test/src/main/sample/data/yaf/indexed/YafIndexed
index 1c38406..d48fa46 100644
--- a/metron-platform/metron-integration-test/src/main/sample/data/yaf/indexed/YafIndexed
+++ b/metron-platform/metron-integration-test/src/main/sample/data/yaf/indexed/YafIndexed
@@ -1,10 +1,10 @@
-{"adapter.threatinteladapter.end.ts":"1457102731219","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa001","index.elasticsearchwriter.ts":"1457102731220","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731206","adapter.hostfromjsonlistadapter.begin.ts":"1457102731185","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":44,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731185","threatintelsplitterbolt.splitter.ts":"1457102731207","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731210","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AS","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731220","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.host.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":10000000,"index.elasticsearchwriter.ts":"1457102731221","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731208","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitter.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988502,"adapter.threatinteladapter.begin.ts":"1457102731219","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731221","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":37299,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988502,"risn":0,"end_time":1453994988502,"is_alert":"true","source.type":"yaf","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":37299,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":312,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitter.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988504,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988504,"enrichments.host.dip.known_info.asset_value":"important","is_alert":"true","source.type":"yaf","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731211","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":56303,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988504,"risn":0,"end_time":1453994988504,"is_alert":"true","source.type":"yaf","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":56303,"rpkt":0,"original_string":"2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":84,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988506,"adapter.threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988506,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988506,"enrichments.host.dip.known_info.asset_value":"important","is_alert":"true","source.type":"yaf","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fca","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":60,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988508,"adapter.threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"S","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988508,"risn":0,"end_time":1453994988508,"source.type":"yaf","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"end_time":1453994988512,"source.type":"yaf","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":148,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"end_time":1453994988512,"source.type":"yaf","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731225","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.host.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
-{"adapter.threatinteladapter.end.ts":"1457102731226","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":604,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731213","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988562,"adapter.threatinteladapter.begin.ts":"1457102731226","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731226","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988562,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988562,"enrichments.host.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731219","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":"22efa001","index.elasticsearchwriter.ts":"1457102731220","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731206","adapter.hostfromjsonlistadapter.begin.ts":"1457102731185","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":44,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731185","threatintelsplitterbolt.splitter.ts":"1457102731207","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731210","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AS","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731220","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.host.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":10000000,"index.elasticsearchwriter.ts":"1457102731221","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731208","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitter.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988502,"adapter.threatinteladapter.begin.ts":"1457102731219","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731221","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":37299,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988502,"risn":0,"end_time":1453994988502,"is_alert":"true","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":37299,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":312,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitter.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988504,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988504,"enrichments.host.dip.known_info.asset_value":"important","is_alert":"true","source.type":"yaf","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731211","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":56303,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988504,"risn":0,"end_time":1453994988504,"is_alert":"true","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":56303,"rpkt":0,"original_string":"2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":84,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988506,"adapter.threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988506,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988506,"enrichments.host.dip.known_info.asset_value":"important","is_alert":"true","source.type":"yaf","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fca","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":60,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988508,"adapter.threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"S","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988508,"risn":0,"end_time":1453994988508,"source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"end_time":1453994988512,"source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test latitude,test longitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":148,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"end_time":1453994988512,"source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731225","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.host.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731226","enrichments.geo.dip.location_point":"test latitude,test longitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":604,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731213","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988562,"adapter.threatinteladapter.begin.ts":"1457102731226","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731226","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test latitude,test longitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988562,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988562,"enrichments.host.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index 2630ef3..fc25e09 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -32,11 +32,6 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.metron</groupId>
- <artifactId>metron-pcap</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${global_hadoop_version}</version>
@@ -100,7 +95,16 @@
<artifactId>guava</artifactId>
<version>${global_hbase_guava_version}</version>
</dependency>
-
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.2.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.2.3</version>
+ </dependency>
<dependency>
<groupId>io.thekraken</groupId>
<artifactId>grok</artifactId>
@@ -129,6 +133,12 @@
<version>${project.parent.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.github.fge</groupId>
+ <artifactId>json-schema-validator</artifactId>
+ <version>2.2.6</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<reporting>
<plugins>
@@ -183,6 +193,12 @@
<goal>shade</goal>
</goals>
<configuration>
+ <relocations>
+ <relocation>
+ <pattern>com.fasterxml.jackson.core</pattern>
+ <shadedPattern>com.fasterxml.jackson.core.metron.parsers</shadedPattern>
+ </relocation>
+ </relocations>
<artifactSet>
<excludes>
<exclude>storm:storm-core:*</exclude>
diff --git a/metron-platform/metron-pcap/pom.xml b/metron-platform/metron-pcap/pom.xml
index 6a45617..29fdbab 100644
--- a/metron-platform/metron-pcap/pom.xml
+++ b/metron-platform/metron-pcap/pom.xml
@@ -31,6 +31,21 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${global_jackson_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${global_jackson_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${global_jackson_version}</version>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${global_guava_version}</version>
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java
index 8aa201e..31e8d49 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrEnrichmentIntegrationTest.java
@@ -19,6 +19,7 @@
import com.google.common.base.Function;
import org.apache.metron.common.configuration.Configurations;
+import org.apache.metron.common.interfaces.FieldNameConverter;
import org.apache.metron.integration.EnrichmentIntegrationTest;
import org.apache.metron.integration.ComponentRunner;
import org.apache.metron.integration.InMemoryComponent;
@@ -39,6 +40,11 @@
public class SolrEnrichmentIntegrationTest extends EnrichmentIntegrationTest {
private String collection = "metron";
+ private FieldNameConverter fieldNameConverter = fieldName -> fieldName;
+ @Override
+ public FieldNameConverter getFieldNameConverter() {
+ return fieldNameConverter;
+ }
@Override
public InMemoryComponent getSearchComponent(final Properties topologyProperties) throws Exception {
diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml
index 14bf06a..9267a43 100644
--- a/metron-platform/pom.xml
+++ b/metron-platform/pom.xml
@@ -38,7 +38,7 @@
<global_hadoop_version>2.7.1</global_hadoop_version>
<global_hbase_version>1.1.1</global_hbase_version>
<global_flume_version>1.5.2</global_flume_version>
- <global_elasticsearch_version>1.7.4</global_elasticsearch_version>
+ <global_elasticsearch_version>2.3.3</global_elasticsearch_version>
<global_json_simple_version>1.1.1</global_json_simple_version>
<global_metrics_version>3.0.2</global_metrics_version>
<global_junit_version>4.4</global_junit_version>
@@ -51,6 +51,7 @@
<global_solr_version>5.2.1</global_solr_version>
<global_mockito_version>1.10.19</global_mockito_version>
<global_shade_version>2.4.3</global_shade_version>
+ <global_jackson_version>2.7.4</global_jackson_version>
</properties>
<licenses>
<license>
@@ -60,10 +61,8 @@
</license>
</licenses>
<modules>
-
<module>metron-common</module>
<module>metron-enrichment</module>
- <module>metron-elasticsearch</module>
<module>metron-solr</module>
<module>metron-parsers</module>
<module>metron-pcap-backend</module>
@@ -73,6 +72,8 @@
<module>metron-test-utilities</module>
<module>metron-api</module>
<module>metron-hbase</module>
+ <module>elasticsearch-shaded</module>
+ <module>metron-elasticsearch</module>
</modules>
<dependencyManagement>
<dependencies>
