METRON-2111: Update public web site to point at 0.7.1 new release
diff --git a/current-book/CONTRIBUTING.html b/current-book/CONTRIBUTING.html
index 1517586..6fdab0c 100644
--- a/current-book/CONTRIBUTING.html
+++ b/current-book/CONTRIBUTING.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/CONTRIBUTING.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/CONTRIBUTING.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – How To Contribute</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">How To Contribute</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/Upgrading.html b/current-book/Upgrading.html
index db9c0dd..eedcbda 100644
--- a/current-book/Upgrading.html
+++ b/current-book/Upgrading.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/Upgrading.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/Upgrading.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Upgrading</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Upgrading</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -93,6 +91,22 @@
<p><a name="Upgrading"></a></p>
<p>This document constitutes a per-version listing of changes of configuration which are non-backwards compatible.</p>
<div class="section">
+<h2><a name="a0.7.0_to_0.7.1"></a>0.7.0 to 0.7.1</h2>
+<div class="section">
+<h3><a name="METRON-2100:_Update_developer_documentation_for_full_dev_management_UI_parser_aggregation_feature_gap"></a><a class="externalLink" href="https://issues.apache.org/jira/browse/METRON-2100">METRON-2100: Update developer documentation for full dev management UI parser aggregation feature gap</a></h3>
+<p>The original full_dev environment change was actually introduced in Metron 0.7.0. This Jira addresses missing user documentation for the Management UI feature gap for parser aggregation. See <a href="metron-deployment/index.html#Parser_Aggregation_Feature">Parser Aggregation Feature</a> for more details on how to work with and configure parsers with this feature change enabled in full_dev.</p></div>
+<div class="section">
+<h3><a name="METRON-2053:_Refactor_metron-enrichment_to_decouple_Storm_dependencies"></a><a class="externalLink" href="https://issues.apache.org/jira/browse/METRON-2053">METRON-2053: Refactor metron-enrichment to decouple Storm dependencies</a></h3>
+<p><tt>org.apache.metron.enrichment.writer.SimpleHbaseEnrichmentWriter</tt> has had its packaged changed to <tt>org.apache.metron.writer.hbase.SimpleHbaseEnrichmentWriter</tt>. It has also been moved from the <tt>metron-platform/metron-enrichment</tt> module to a more appropriate home in <tt>metron-platform/metron-writer</tt>.</p></div>
+<div class="section">
+<h3><a name="METRON-1929:_Build_GET_ASN_Stellar_function"></a><a class="externalLink" href="https://issues.apache.org/jira/browse/METRON-1929">METRON-1929: Build GET_ASN Stellar function</a></h3>
+<p>The script for <tt>geo_enrichment_load.sh</tt> has been renamed, and now is <tt>maxmind_enrichment_load.sh</tt>. A couple changes should happen for users who are upgrading.</p>
+<ul>
+
+<li>The MaxMind GeoLite2 ASN database should be loaded onto HDFS at /apps/metron/asn/default/GeoLite2-ASN.tar.gz OR the global configuration property <tt>asn.hdfs.file</tt> can be set to point to a custom HDFS location.</li>
+<li>Any custom scripts or tasks that use this script should be updated. In addition, this updated script also retrieves the GeoLite2 ASN database. The <tt>-ra</tt> flag can be used to provide a custom location for this database if offline install is needed. Otherwise, it will retrieve the latest from MaxMind.</li>
+</ul></div></div>
+<div class="section">
<h2><a name="a0.6.0_to_0.7.0"></a>0.6.0 to 0.7.0</h2>
<div class="section">
<h3><a name="METRON-1834:_Migrate_Elasticsearch_from_TransportClient_to_new_Java_REST_API"></a><a class="externalLink" href="https://issues.apache.org/jira/browse/METRON-1834">METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API</a></h3>
diff --git a/current-book/index.html b/current-book/index.html
index cf4cc2d..17ea009 100644
--- a/current-book/index.html
+++ b/current-book/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Apache Metron</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Apache Metron</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -185,7 +183,7 @@
<p>Some high level links to the relevant subparts of the architecture, for more information:</p>
<ul>
-<li><a href="metron-platform/metron-parsers/index.html">Parsers</a> : Parsing data from kafka into the Metron data model and passing it downstream to Enrichment.</li>
+<li><a href="metron-platform/metron-parsers-common/index.html">Parsers</a> : Parsing data from kafka into the Metron data model and passing it downstream to Enrichment.</li>
<li><a href="metron-platform/metron-enrichment/index.html">Enrichment</a> : Enriching data post-parsing and providing the ability to tag a message as an alert and assign a risk triage level via a custom rule language.</li>
<li><a href="metron-platform/metron-indexing/index.html">Indexing</a> : Indexing the data post-enrichment into HDFS, Elasticsearch or Solr.</li>
</ul>
diff --git a/current-book/metron-analytics/index.html b/current-book/metron-analytics/index.html
index 77dbf24..941db5e 100644
--- a/current-book/metron-analytics/index.html
+++ b/current-book/metron-analytics/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Analytics</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Analytics</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -59,9 +59,7 @@
<li><a href="../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-analytics/metron-maas-service/index.html b/current-book/metron-analytics/metron-maas-service/index.html
index 99cfb00..fa5892e 100644
--- a/current-book/metron-analytics/metron-maas-service/index.html
+++ b/current-book/metron-analytics/metron-maas-service/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-maas-service/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-maas-service/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Model Management Infrastructure</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Model Management Infrastructure</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -59,9 +59,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-analytics/metron-profiler-client/index.html b/current-book/metron-analytics/metron-profiler-client/index.html
index 1445e79..9a22842 100644
--- a/current-book/metron-analytics/metron-profiler-client/index.html
+++ b/current-book/metron-analytics/metron-profiler-client/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-profiler-client/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-profiler-client/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Profiler Client</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Profiler Client</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -59,9 +59,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-analytics/metron-profiler-common/index.html b/current-book/metron-analytics/metron-profiler-common/index.html
index f2c802e..59ea1a1 100644
--- a/current-book/metron-analytics/metron-profiler-common/index.html
+++ b/current-book/metron-analytics/metron-profiler-common/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-profiler-common/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-profiler-common/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Profiler</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Profiler</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -59,9 +59,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-analytics/metron-profiler-repl/index.html b/current-book/metron-analytics/metron-profiler-repl/index.html
index eafb1dc..4c47258 100644
--- a/current-book/metron-analytics/metron-profiler-repl/index.html
+++ b/current-book/metron-analytics/metron-profiler-repl/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-profiler-repl/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-profiler-repl/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Profiler for the Stellar REPL</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Profiler for the Stellar REPL</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -59,9 +59,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-analytics/metron-profiler-spark/index.html b/current-book/metron-analytics/metron-profiler-spark/index.html
index 80eb19c..730b70c 100644
--- a/current-book/metron-analytics/metron-profiler-spark/index.html
+++ b/current-book/metron-analytics/metron-profiler-spark/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-profiler-spark/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-profiler-spark/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Profiler for Spark</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Profiler for Spark</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -59,9 +59,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -164,6 +162,15 @@
</li>
<li>
+<p>Copy the <tt>hbase-site.xml</tt> file from <tt>/etc/hbase/conf</tt> to <tt>/etc/spark2/conf</tt>. It is advised to create a symlink to avoid the duplication of file, also to keep consistency between files while config updates.</p>
+
+<div>
+<div>
+<pre class="source">ln -s /etc/hbase/conf/hbase-site.xml /etc/spark2/conf/hbase-site.xml
+</pre></div></div>
+</li>
+<li>
+
<p>Review the Batch Profiler’s properties located at <tt>$METRON_HOME/config/batch-profiler.properties</tt>. See <a href="#Configuring_the_Profiler">Configuring the Profiler</a> for more information on these properties.</p>
</li>
<li>
diff --git a/current-book/metron-analytics/metron-profiler-storm/index.html b/current-book/metron-analytics/metron-profiler-storm/index.html
index 9c97ac8..31d0dfa 100644
--- a/current-book/metron-analytics/metron-profiler-storm/index.html
+++ b/current-book/metron-analytics/metron-profiler-storm/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-profiler-storm/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-profiler-storm/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Profiler for Storm</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Profiler for Storm</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -59,9 +59,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-analytics/metron-statistics/HLLP.html b/current-book/metron-analytics/metron-statistics/HLLP.html
index e7434db..528dac2 100644
--- a/current-book/metron-analytics/metron-statistics/HLLP.html
+++ b/current-book/metron-analytics/metron-statistics/HLLP.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-statistics/HLLP.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-statistics/HLLP.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – HyperLogLogPlus</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">HyperLogLogPlus</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -63,9 +63,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-analytics/metron-statistics/index.html b/current-book/metron-analytics/metron-statistics/index.html
index 25f92e5..f00bd84 100644
--- a/current-book/metron-analytics/metron-statistics/index.html
+++ b/current-book/metron-analytics/metron-statistics/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-statistics/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-statistics/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Statistics and Mathematical Functions</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Statistics and Mathematical Functions</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -63,9 +63,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-contrib/metron-docker/index.html b/current-book/metron-contrib/metron-docker/index.html
index 2bd84d2..aad9134 100644
--- a/current-book/metron-contrib/metron-docker/index.html
+++ b/current-book/metron-contrib/metron-docker/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-contrib/metron-docker/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-contrib/metron-docker/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Docker</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Docker</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li class="active"><a href="#"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -212,7 +210,7 @@
</ul>
<div class="section">
<h3><a name="Deploy_a_new_parser_class"></a>Deploy a new parser class</h3>
-<p>After adding a new parser to metron-parsers, build Metron from the top level directory:</p>
+<p>After adding a new parser to metron-parsers-common, build Metron from the top level directory:</p>
<div>
<div>
diff --git a/current-book/metron-contrib/metron-performance/index.html b/current-book/metron-contrib/metron-performance/index.html
index 675f5c3..f7d95ef 100644
--- a/current-book/metron-contrib/metron-performance/index.html
+++ b/current-book/metron-contrib/metron-performance/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-contrib/metron-performance/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-contrib/metron-performance/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Performance Utilities</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Performance Utilities</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li class="active"><a href="#"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/Kerberos-ambari-setup.html b/current-book/metron-deployment/Kerberos-ambari-setup.html
index 7f2541f..087b5f6 100644
--- a/current-book/metron-deployment/Kerberos-ambari-setup.html
+++ b/current-book/metron-deployment/Kerberos-ambari-setup.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/Kerberos-ambari-setup.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/Kerberos-ambari-setup.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Setting Up Kerberos in Vagrant Full Dev</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Setting Up Kerberos in Vagrant Full Dev</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -63,9 +63,7 @@
<li><a href="../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/Kerberos-manual-setup.html b/current-book/metron-deployment/Kerberos-manual-setup.html
index 89e052b..9de283b 100644
--- a/current-book/metron-deployment/Kerberos-manual-setup.html
+++ b/current-book/metron-deployment/Kerberos-manual-setup.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/Kerberos-manual-setup.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/Kerberos-manual-setup.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Kerberos Setup</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Kerberos Setup</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -63,9 +63,7 @@
<li><a href="../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -118,6 +116,7 @@
<li><a href="#Push_Data">Push Data</a></li>
<li><a href="#More_Information">More Information</a></li>
<li><a href="#X-Pack">Elasticseach X-Pack</a></li>
+<li><a href="#TGT_Ticket_Renew">TGT Ticket Renew</a></li>
</ul>
<div class="section">
<h2><a name="Setup"></a>Setup</h2>
@@ -829,7 +828,7 @@
</li>
<li>
-<p>Restart the metron-rest service, and make sure the elasticsearch-xpack-shaded-5.6.2.jar is in the METRON_REST_CLASSPATH when the metron-rest starts.</p>
+<p>Restart the metron-rest service, and make sure the elasticsearch-xpack-shaded-5.6.14.jar is in the METRON_REST_CLASSPATH when the metron-rest starts.</p>
</li>
</ol>
<p>Once you’ve performed these steps, you should be able to start seeing data in your ES indexes.</p>
@@ -878,6 +877,47 @@
<div class="section">
<h4><a name="Solution"></a>Solution</h4>
<p>This can occur when an HDFS Client is not installed on the Storm worker nodes. This might occur on any Storm worker node where an HDFS Client is not installed. Installing the HDFS Client on all Storm worker nodes should resolve the problem.</p></div></div></div>
+<div class="section">
+<h2><a name="TGT_Ticket_Renew"></a>TGT Ticket Renew</h2>
+<p>Apache Storm doesn’t handle automatic TGT ticket renewal for their running topologies. Instead, it is left up to the operations team deploying the Storm topologies in a Kerberized environment to manage this themselves. We’ve included a Python script that can be setup with a cron process to automatically manage the renewal process for you. The script should be run on an interval that is shorter than the renew_lifetime configured for your TGT.</p>
+<div class="section">
+<h3><a name="Setup_Instructions"></a>Setup Instructions</h3>
+<p>Run the following on a node with a Storm and Metron client installed. We need python 2.7 via virtualenv for this to work correctly.</p>
+
+<div>
+<div>
+<pre class="source"># run yum commands as root
+for item in epel-release centos-release-scl "@Development tools" python27 python27-scldevel python27-python-virtualenv libselinux-python; do yum install -y $item; done
+sudo yum install -y gcc krb5-devel python-devel
+sudo yum install -y libffi libffi-devel
+sudo yum install -y python-cffi
+sudo yum install -y openssl-devel
+# setup python with metron user
+su - metron
+export PYTHON27_HOME=/opt/rh/python27/root
+export LD_LIBRARY_PATH="/opt/rh/python27/root/usr/lib64"
+mkdir project_dir
+cd project_dir
+${PYTHON27_HOME}/usr/bin/virtualenv venv
+source venv/bin/activate
+pip install --upgrade setuptools==18.5
+pip install requests-kerberos
+</pre></div></div>
+
+<p>The script <tt>$METRON_HOME/bin/tgt_renew.py</tt> takes two arguments:</p>
+<ul>
+
+<li>arg1 = host:port for Storm UI server</li>
+<li>arg2 = topology owner - typically “metron” for a kerberized cluster with metron topologies</li>
+</ul>
+<p>Execute it like the following example:</p>
+
+<div>
+<div>
+<pre class="source"># run as the metron user
+su - metron
+python $METRON_HOME/bin/tgt_renew.py node1:8744 metron
+</pre></div></div></div></div>
</div>
</div>
</div>
diff --git a/current-book/metron-deployment/amazon-ec2/index.html b/current-book/metron-deployment/amazon-ec2/index.html
index 200f02d..a53a48b 100644
--- a/current-book/metron-deployment/amazon-ec2/index.html
+++ b/current-book/metron-deployment/amazon-ec2/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/amazon-ec2/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/amazon-ec2/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Apache Metron on Amazon EC2</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Apache Metron on Amazon EC2</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -63,9 +63,7 @@
<li><a href="../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/ansible/index.html b/current-book/metron-deployment/ansible/index.html
index 544c186..95959e1 100644
--- a/current-book/metron-deployment/ansible/index.html
+++ b/current-book/metron-deployment/ansible/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Ansible Assets</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Ansible Assets</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -67,9 +67,7 @@
<li><a href="../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/ansible/roles/index.html b/current-book/metron-deployment/ansible/roles/index.html
index cd3d63c..be16ab5 100644
--- a/current-book/metron-deployment/ansible/roles/index.html
+++ b/current-book/metron-deployment/ansible/roles/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/roles/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/roles/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Ansible Roles</title>
<link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Ansible Roles</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -74,9 +74,7 @@
<li><a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/ansible/roles/opentaxii/index.html b/current-book/metron-deployment/ansible/roles/opentaxii/index.html
index 40a014d..4d65ecc 100644
--- a/current-book/metron-deployment/ansible/roles/opentaxii/index.html
+++ b/current-book/metron-deployment/ansible/roles/opentaxii/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/roles/opentaxii/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/roles/opentaxii/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – OpenTAXII</title>
<link rel="stylesheet" href="../../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">OpenTAXII</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -74,9 +74,7 @@
<li><a href="../../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/ansible/roles/pcap_replay/index.html b/current-book/metron-deployment/ansible/roles/pcap_replay/index.html
index f569f7d..31ddbba 100644
--- a/current-book/metron-deployment/ansible/roles/pcap_replay/index.html
+++ b/current-book/metron-deployment/ansible/roles/pcap_replay/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/roles/pcap_replay/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/roles/pcap_replay/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Pcap Replay</title>
<link rel="stylesheet" href="../../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Pcap Replay</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -74,9 +74,7 @@
<li><a href="../../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/ansible/roles/sensor-stubs/index.html b/current-book/metron-deployment/ansible/roles/sensor-stubs/index.html
index 8b75316..821fcb1 100644
--- a/current-book/metron-deployment/ansible/roles/sensor-stubs/index.html
+++ b/current-book/metron-deployment/ansible/roles/sensor-stubs/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/roles/sensor-stubs/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/roles/sensor-stubs/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Sensor Stubs</title>
<link rel="stylesheet" href="../../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Sensor Stubs</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -74,9 +74,7 @@
<li><a href="../../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/ansible/roles/sensor-test-mode/index.html b/current-book/metron-deployment/ansible/roles/sensor-test-mode/index.html
index 9716def..1376121 100644
--- a/current-book/metron-deployment/ansible/roles/sensor-test-mode/index.html
+++ b/current-book/metron-deployment/ansible/roles/sensor-test-mode/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/roles/sensor-test-mode/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/ansible/roles/sensor-test-mode/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Sensor Test Mode</title>
<link rel="stylesheet" href="../../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Sensor Test Mode</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -74,9 +74,7 @@
<li><a href="../../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/development/centos6/index.html b/current-book/metron-deployment/development/centos6/index.html
index efb7a53..cd59a4a 100644
--- a/current-book/metron-deployment/development/centos6/index.html
+++ b/current-book/metron-deployment/development/centos6/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/development/centos6/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/development/centos6/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron on CentOS 6</title>
<link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron on CentOS 6</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -69,9 +69,7 @@
<li><a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/development/fastcapa/index.html b/current-book/metron-deployment/development/fastcapa/index.html
index ca9a216..24acbb4 100644
--- a/current-book/metron-deployment/development/fastcapa/index.html
+++ b/current-book/metron-deployment/development/fastcapa/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/development/fastcapa/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/development/fastcapa/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Fastcapa Test Environment</title>
<link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Fastcapa Test Environment</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -69,9 +69,7 @@
<li><a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/development/index.html b/current-book/metron-deployment/development/index.html
index 7152a3f..0167095 100644
--- a/current-book/metron-deployment/development/index.html
+++ b/current-book/metron-deployment/development/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/development/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/development/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Development Environments</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Development Environments</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -69,9 +69,7 @@
<li><a href="../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/development/ubuntu14/index.html b/current-book/metron-deployment/development/ubuntu14/index.html
index 3ddc804..39be1c2 100644
--- a/current-book/metron-deployment/development/ubuntu14/index.html
+++ b/current-book/metron-deployment/development/ubuntu14/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/development/ubuntu14/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/development/ubuntu14/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron on Ubuntu 14</title>
<link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron on Ubuntu 14</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -69,9 +69,7 @@
<li><a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/index.html b/current-book/metron-deployment/index.html
index 5a28ee4..06b888f 100644
--- a/current-book/metron-deployment/index.html
+++ b/current-book/metron-deployment/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – </title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active "></li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -63,9 +63,7 @@
<li><a href="../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -174,7 +172,49 @@
<p>To deploy Metron in a VM running on your computer, follow the instructions at <a href="development/centos6/index.html">development/centos6</a>.</p></div>
<div class="section">
<h4><a name="How_do_I_address_services_crashing_when_running_Metron_on_a_single_VM.3F"></a>How do I address services crashing when running Metron on a single VM?</h4>
-<p>We recommend looking at Ambari and shutting down any services you may not be using. For example, we recommend turning off Metron Profiler, as this commonly causes REST services to crash when running on a single VM.</p></div></div></div>
+<p>We recommend looking at Ambari and shutting down any services you may not be using. For example, we recommend turning off Metron Profiler, as this commonly causes REST services to crash when running on a single VM.</p></div></div>
+<div class="section">
+<h3><a name="Parser_Aggregation_Feature"></a>Parser Aggregation Feature</h3>
+<p>The <a href="../metron-platform/metron-parsing/metron-parsing-storm/index.html#Parser_Aggregation">Parser Aggregation</a> feature does not currently exist in the management UI. In order to address resource limitations in the full dev development environments, bro, yaf, and snort have been aggregated into a single parser topology. However, the Management UI is not currently able to display its status until the feature is added. Aggregated parsers can still be created via Ambari and the command line scripts.</p>
+<p>Here are some tips for working with parser aggregation while the UI feature is being developed.</p>
+<ul>
+
+<li>
+
+<p><b>How are parsers picked up by the UI?:</b> This is based entirely on what is currently stored in the Zookeeper configs. See <a href="../metron-platform/metron-common/index.html#Management_Utility">Management Utility</a> “DUMP” option with “-c PARSER” to see all of what is currently loaded. The management UI does not update the configurations stored locally on disk, so Zookeeper is the source of truth.</p>
+</li>
+<li>
+
+<p><b>Removing an existing aggregation:</b> In the <a class="externalLink" href="http://node1:8080">Ambari UI</a> click on the Metron serice and select “Metron Parsers.” Select “stop” from the dropdown for the parser component. Click “back,” “configs,” and then navigate to “Parsers.” In the text field option labeled “parsers”. Remove the double quotes from around the listed parsers. Save and choose “Restart” when prompted. This will deploy three individual parsers rather than a single aggregated parser: bro, snort, and yaf. Be aware, you may need to shut down other topologies to free up resources so that you can run the parsers without aggregation. Stopping the profiler, pcap, or batch_indexing are a few options that will still allow data to pass through the system end-to-end.</p>
+</li>
+<li>
+
+<p><b>Managing parser lifecycle:</b> Starting and stopping parsers in the management UI will in no way affect a parser running as aggregated. The exception to this is if you create a parser via the management UI that has the same name as the aggregation, e.g. “bro__snort__yaf.” We recommend against this. It will appear as thought you now have the ability to manage the aggregated parser now, but you will only be able to start/stop it.</p>
+</li>
+<li>
+
+<p><b>Editing parser configuration:</b> In order to modify the aggregated parsers’ configurations, you will need to first pull all of the configuration from Zookeeper to the local configuration directory by executing the following commands</p>
+
+<div>
+<div>
+<pre class="source">source /etc/default/metron
+$METRON_HOME/bin/zk_load_configs.sh -m PULL -o ${METRON_HOME}/config/zookeeper -z $ZOOKEEPER -f
+</pre></div></div>
+
+<p>Make your changes to an individual parser’s configuration json, e.g. <tt>${METRON_HOME}/config/zookeeper/parsers/bro.json</tt>, save locally, and then push them back up to Zookeeper</p>
+
+<div>
+<div>
+<pre class="source">$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i $METRON_HOME/config/zookeeper/ -z $ZOOKEEPER
+</pre></div></div>
+
+<p>See <a href="../metron-platform/metron-common/index.html#Management_Utility">Management Utility</a> for more detail.</p>
+</li>
+<li>
+
+<p><b>Other gotchas:</b> Stop the aggregated parsers in Ambari before removing or adding grouping quotes. Otherwise, you will end up with both the individual parsers and the aggregated topology running concurrently. Ambari only manages the parser topology lifecycle via the current parser name list provided, so changing that list removes Ambari’s ability to reference the old topology names.</p>
+</li>
+</ul></div></div>
<div class="section">
<h2><a name="How_do_I_build_RPM_packages.3F"></a>How do I build RPM packages?</h2>
<p>This provides RPM packages that allow you to install Metron on an RPM-based operating system like CentOS.</p>
diff --git a/current-book/metron-deployment/other-examples/index.html b/current-book/metron-deployment/other-examples/index.html
index d18a58b..f31abd4 100644
--- a/current-book/metron-deployment/other-examples/index.html
+++ b/current-book/metron-deployment/other-examples/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/other-examples/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/other-examples/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Other Example Deployments</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Other Example Deployments</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -67,9 +67,7 @@
<li><a href="../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html b/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html
index 56622a5..311c451 100644
--- a/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html
+++ b/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – </title>
<link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active "></li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -67,9 +67,7 @@
<li><a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html b/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html
index 93b2902..e2334ee 100644
--- a/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html
+++ b/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/ambari/elasticsearch-mpack/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/ambari/elasticsearch-mpack/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – </title>
<link rel="stylesheet" href="../../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active "></li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -68,9 +68,7 @@
<li><a href="../../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/packaging/ambari/index.html b/current-book/metron-deployment/packaging/ambari/index.html
index 51336b6..897b10c 100644
--- a/current-book/metron-deployment/packaging/ambari/index.html
+++ b/current-book/metron-deployment/packaging/ambari/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/ambari/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/ambari/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Ambari Management Pack Development</title>
<link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Ambari Management Pack Development</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -68,9 +68,7 @@
<li><a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -194,7 +192,7 @@
<li><tt>metainfo.xml</tt>
<ul>
-<li>Defines the METRON service, along with required packages, services, etc.</li>
+<li>Defines the METRON service, along with required packages, services, etc. If you need to have the MPack install a new package (e.g. RPM, DEB), add it here.</li>
</ul>
</li>
<li><tt>service_advisor.py</tt>
@@ -767,7 +765,7 @@
<div>
<div>
-<pre class="source">metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2
+<pre class="source">metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.14
</pre></div></div>
</li>
<li>
@@ -777,14 +775,14 @@
<div>
<div>
-<pre class="source"><version>5.6.2</version>
+<pre class="source"><version>5.6.14</version>
...
<osSpecifics>
<osSpecific>
<osFamily>any</osFamily>
<packages>
<package>
- <name>elasticsearch-5.6.2</name>
+ <name>elasticsearch-5.6.14</name>
</package>
</packages>
</osSpecific>
@@ -809,7 +807,7 @@
<div>
<div>
-<pre class="source">metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2
+<pre class="source">metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.14
</pre></div></div>
</li>
<li>
@@ -833,8 +831,8 @@
<div>
<div>
<pre class="source"><name>ELASTICSEARCH</name>
-<version>5.6.2</version>
-<extends>common-services/ELASTICSEARCH/5.6.2</extends>
+<version>5.6.14</version>
+<extends>common-services/ELASTICSEARCH/5.6.14</extends>
</pre></div></div>
</li>
</ol></div>
@@ -850,7 +848,7 @@
"service_versions_map": [
{
"service_name" : "ELASTICSEARCH",
- "service_version" : "5.6.2",
+ "service_version" : "5.6.14",
"applicable_stacks" : [
...
]
@@ -881,7 +879,7 @@
<div>
<div>
-<pre class="source">metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2
+<pre class="source">metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.14
</pre></div></div>
</li>
<li>
@@ -891,12 +889,12 @@
<div>
<div>
-<pre class="source"><version>5.6.2</version>
+<pre class="source"><version>5.6.14</version>
...
<packages>
...
<package>
- <name>kibana-5.6.2</name>
+ <name>kibana-5.6.14</name>
</package>
</packages>
</pre></div></div>
@@ -919,7 +917,7 @@
<div>
<div>
-<pre class="source">metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2
+<pre class="source">metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.14
</pre></div></div>
</li>
<li>
@@ -953,8 +951,8 @@
<div>
<div>
<pre class="source"><name>KIBANA</name>
-<version>5.6.2</version>
-<extends>common-services/KIBANA/5.6.2</extends>
+<version>5.6.14</version>
+<extends>common-services/KIBANA/5.6.14</extends>
</pre></div></div>
</li>
</ol></div>
@@ -970,7 +968,7 @@
"service_versions_map": [
{
"service_name" : "KIBANA",
- "service_version" : "5.6.2",
+ "service_version" : "5.6.14",
"applicable_stacks" : [
...
]
diff --git a/current-book/metron-deployment/packaging/ambari/metron-mpack/index.html b/current-book/metron-deployment/packaging/ambari/metron-mpack/index.html
index d5fdf0b..2b33f44 100644
--- a/current-book/metron-deployment/packaging/ambari/metron-mpack/index.html
+++ b/current-book/metron-deployment/packaging/ambari/metron-mpack/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/ambari/metron-mpack/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/ambari/metron-mpack/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – </title>
<link rel="stylesheet" href="../../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active "></li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -68,9 +68,7 @@
<li><a href="../../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -252,11 +250,11 @@
~/dashboard-bulkload.json
</pre></div></div>
-<p>Locate the “create” command for setting the default index by searching for “5.6.2”. Change “create” to “index” so that it modifies the existing value. It should look similar to line 1 below.</p>
+<p>Locate the “create” command for setting the default index by searching for “5.6.14”. Change “create” to “index” so that it modifies the existing value. It should look similar to line 1 below.</p>
<div>
<div>
-<pre class="source">{ "index" : { "_id": "5.6.2", "_type": "config" } }
+<pre class="source">{ "index" : { "_id": "5.6.14", "_type": "config" } }
{"defaultIndex":"AV-S2e81hKs1cXXnFMqN"}
</pre></div></div>
@@ -301,7 +299,8 @@
<p>A custom action is available in Ambari to import Zeppelin dashboards. See the <a href="../../../../metron-platform/metron-indexing/index.html">metron-indexing documentation</a> for more information.</p></div>
<div class="section">
<h4><a name="Offline_Installation"></a>Offline Installation</h4>
-<p>Retrieval of the GeoIP database is the only point during installation that reaches out to the internet. For an offline installation, the URL for the GeoIP database can be manually set to a local path on the file system such as <tt>file:///home/root/geoip/GeoLite2-City.mmdb.gz</tt>.</p>
+<p>Retrieval of the GeoIP and ASN databases (both from MaxMind) is the only point during installation that reaches out to the internet. For an offline installation, the URL for the databases can be manually set to a local path on the file system such as <tt>file:///home/root/geoip/GeoLite2-City.tar.gz</tt>.</p>
+<p>The properties for configuration are <tt>geoip_url</tt> and <tt>asn_url</tt> in the <tt>Enrichment</tt> section.</p>
<p>The RPMs DO NOT reach out to the internet (because there is currently no hosting for them). They look on the local filesystem in <tt>/localrepo</tt>.</p></div>
<div class="section">
<h4><a name="Limitations"></a>Limitations</h4>
diff --git a/current-book/metron-deployment/packaging/docker/ansible-docker/index.html b/current-book/metron-deployment/packaging/docker/ansible-docker/index.html
index 3689ddc..3b63168 100644
--- a/current-book/metron-deployment/packaging/docker/ansible-docker/index.html
+++ b/current-book/metron-deployment/packaging/docker/ansible-docker/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/docker/ansible-docker/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/docker/ansible-docker/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – </title>
<link rel="stylesheet" href="../../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active "></li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -63,9 +63,7 @@
<li><a href="../../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/packaging/docker/deb-docker/index.html b/current-book/metron-deployment/packaging/docker/deb-docker/index.html
index 3b3cc79..34e67a6 100644
--- a/current-book/metron-deployment/packaging/docker/deb-docker/index.html
+++ b/current-book/metron-deployment/packaging/docker/deb-docker/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/docker/deb-docker/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/docker/deb-docker/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – </title>
<link rel="stylesheet" href="../../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active "></li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -63,9 +63,7 @@
<li><a href="../../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/packaging/docker/rpm-docker/index.html b/current-book/metron-deployment/packaging/docker/rpm-docker/index.html
index 6c7e9a1..6293f0f 100644
--- a/current-book/metron-deployment/packaging/docker/rpm-docker/index.html
+++ b/current-book/metron-deployment/packaging/docker/rpm-docker/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/docker/rpm-docker/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/docker/rpm-docker/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – </title>
<link rel="stylesheet" href="../../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active "></li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -63,9 +63,7 @@
<li><a href="../../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-deployment/packaging/packer-build/index.html b/current-book/metron-deployment/packaging/packer-build/index.html
index 706fe21..52b5430 100644
--- a/current-book/metron-deployment/packaging/packer-build/index.html
+++ b/current-book/metron-deployment/packaging/packer-build/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/packer-build/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-deployment/packaging/packer-build/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Build Metron Images</title>
<link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Build Metron Images</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -63,9 +63,7 @@
<li class="active"><a href="#"><span class="none"></span>Packer-build</a></li>
</ul>
</li>
- <li><a href="../../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-interface/index.html b/current-book/metron-interface/index.html
new file mode 100644
index 0000000..213fda7
--- /dev/null
+++ b/current-book/metron-interface/index.html
@@ -0,0 +1,232 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-interface/index.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Metron Interface</title>
+ <link rel="stylesheet" href="../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../css/site.css" />
+ <link rel="stylesheet" href="../css/print.css" media="print" />
+ <script type="text/javascript" src="../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">Metron Interface</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li class="active"><a href="#"><span class="icon-chevron-down"></span>Interface</a>
+ <ul class="nav nav-list">
+ <li><a href="../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
+ <li><a href="../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
+ <li><a href="../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ </ul>
+</li>
+ <li><a href="../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
+ <li><a href="../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Metron Interface</h1>
+<p><a name="Metron_Interface"></a></p>
+<p>Metron interface contains code and assets to support the various web applications in Metron. The existing modules are:</p>
+<ul>
+
+<li>metron-alerts : An Angular application that exposes a way to browse, filter and act on alerts.</li>
+<li>metron-config: An Angular application that allows an administrator to configure and maintain Metron.</li>
+<li>metron-rest: A Spring REST application that supports the UI and exposes Metron features through a REST/json interface.</li>
+<li>metron-rest-client: Model objects used for passing data back and forth in the REST application. A Metron client would use these classes for de/serializing requests and responses.</li>
+</ul>
+<div class="section">
+<h2><a name="Architecture"></a>Architecture</h2>
+<p>The UIs and REST server are all run in separate web containers. The UIs are served from separate <a class="externalLink" href="https://expressjs.com/">Express</a> servers that are configured to proxy REST requests to the REST application. Proxying REST requests satisfies the same-origin browser restriction because all requests go to Express, or the same origin.</p>
+<p>REST requests are handled by a <a class="externalLink" href="https://spring.io/projects/spring-boot">Spring Boot</a> application. A <a class="externalLink" href="https://swagger.io/">Swagger</a> interface is available and served by the REST application.</p>
+<div class="section">
+<h3><a name="Security"></a>Security</h3>
+<p>The UIs depend on REST for authentication. Credentials are passed with <a class="externalLink" href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication">Basic authentication</a> and only REST requests require authentication, static assets are not secured. Once authentication has been performed a session is established and a cookie is stored and used to authenticate subsequent requests.</p></div>
+<div class="section">
+<h3><a name="Request.2FResponse_Flow"></a>Request/Response Flow</h3>
+<p>The following diagram illustrates the flow of data for the various types of requests:</p>
+<p><img src="flow_diagram.png" alt="Flow Diagram" /></p></div></div>
+<div class="section">
+<h2><a name="Architecture_with_Knox"></a>Architecture with Knox</h2>
+<p><a class="externalLink" href="https://knox.apache.org/">Apache Knox</a> is a “REST API and Application Gateway for the Apache Hadoop Ecosystem”. It can be enabled for Metron and provides several security benefits:</p>
+<ul>
+
+<li>All requests go through Knox so same-origin browser restrictions are not a concern.</li>
+<li>Knox, in combination with a firewall, can restrict traffic to always go through Knox. This greatly reduces the security attack surface area of the UIs and REST application.</li>
+<li>Provides access to other common Apache Hadoop services</li>
+<li>Provides a single sign on experience between the UIs and REST application</li>
+<li>All requests can be protected and secured</li>
+</ul>
+<p>We primarily use Knox’s proxying and authentication services. Knox acts as a reverse proxy for all UIs and the REST application.</p>
+<div class="section">
+<h3><a name="Knox_Security"></a>Knox Security</h3>
+<p>With Knox enabled, Knox now handles authentication when accessing the UIs and REST together. Basic authentication is still an option for making requests directly to the REST application. Any request to the UIs must go through Knox first and contain the proper security token.<br />
+If a valid token is not found, Knox will redirect to the Knox SSO login form. Once a valid token is found, Knox will then redirect to the original url and the request will be forwarded on. Accessing the REST application through Knox also follows this pattern. The UIs make REST requests this way with Knox enabled since they no longer depend on Express to proxy requests. The context path now determines which type of request it is rather than the host and port.</p>
+<p>REST still requires authentication so a filter is provided that can validate a Knox token using token properties and a Knox public key. The REST application also supports Basic authentication. Since both Knox and the REST application should use the same authentication mechanism, LDAP authentication is required for the REST application.</p>
+<p>Roles are mapped directly to LDAP groups when Knox is enabled for REST. LDAP group names are converted to upper case and prepended with “ROLE_”. For example, if a user’s groups in LDAP were “user” and “admin”, the corresponding roles in REST with Knox enabled would be “ROLE_USER” and “ROLE_ADMIN”.</p></div>
+<div class="section">
+<h3><a name="Knox_Request.2FResponse_Flow"></a>Knox Request/Response Flow</h3>
+<p>The following diagram illustrates the flow of data for the various types of requests when Knox is enabled:</p>
+<p><img src="knox_flow_diagram.png" alt="Knox Flow Diagram" /></p>
+<p>Note how the flow diagrams for Static asset requests and Rest requests (through Knox) are identical.</p></div></div>
+<div class="section">
+<h2><a name="Enabling_Knox_for_Metron"></a>Enabling Knox for Metron</h2>
+<p>Follow the instructions in the next 3 sections to enable Knox for Metron. The new Knox urls will be similar to (substitute the Knox host/port in your environment for <tt>node1:8443</tt>):</p>
+<ul>
+
+<li>Metron Alerts UI - <a class="externalLink" href="https://node1:8443/gateway/metron/metron-alerts/">https://node1:8443/gateway/metron/metron-alerts/</a></li>
+<li>Metron Management UI - <a class="externalLink" href="https://node1:8443/gateway/metron/metron-management/sensors">https://node1:8443/gateway/metron/metron-management/sensors</a></li>
+<li>Metron REST - <a class="externalLink" href="https://node1:8443/gateway/metron/metron-rest/swagger-ui.html">https://node1:8443/gateway/metron/metron-rest/swagger-ui.html</a></li>
+</ul>
+<div class="section">
+<h3><a name="Install_Metron_Clients"></a>Install Metron Clients</h3>
+<p>The Metron Client component in Ambari is responsible for installing the service definition and topology files in the appropriate Knox directories. These files are installed whenever the Metron Client component(s) are installed, started or restarted AND the <tt>Knox Enabled</tt> Metron Ambari property is set to true (under the <tt>Security</tt> tab in the Metron Config section). Ambari calls the script at <tt>$METRON_HOME/bin/install_metron_knox.sh</tt> which installs the following files:</p>
+<ul>
+
+<li><tt>$KNOX_HOME/conf/topologies/metron.xml</tt></li>
+<li><tt>$KNOX_HOME/conf/topologies/metronsso.xml</tt></li>
+<li><tt>$KNOX_HOME/data/services/metron-alerts/$METRON_VERSION/rewrite.xml</tt></li>
+<li><tt>$KNOX_HOME/data/services/metron-alerts/$METRON_VERSION/service.xml</tt></li>
+<li><tt>$KNOX_HOME/data/services/metron-management/$METRON_VERSION/rewrite.xml</tt></li>
+<li><tt>$KNOX_HOME/data/services/metron-management/$METRON_VERSION/service.xml</tt></li>
+<li><tt>$KNOX_HOME/data/services/metron-rest/$METRON_VERSION/rewrite.xml</tt></li>
+<li><tt>$KNOX_HOME/data/services/metron-rest/$METRON_VERSION/service.xml</tt></li>
+</ul>
+<p>A Metron Client should be installed anywhere a Knox Gateway is installed. It is not strictly required but Metron will not be available through any Knox Gateways that do not have these files installed.</p></div>
+<div class="section">
+<h3><a name="Enable_Knox_for_Metron_in_Ambari"></a>Enable Knox for Metron in Ambari</h3>
+<p>After Metron Client components have been installed on the appropriate hosts, there are a couple settings that need to be changed in the Ambari “Security” tab.<br />
+First the Knox SSO public key needs to be read from Knox. Run the following command on a Knox Gateway to get the key:</p>
+
+<div>
+<div>
+<pre class="source">openssl s_client -connect node1:8443 < /dev/null | openssl x509 | grep -v 'CERTIFICATE' | paste -sd "" -
+</pre></div></div>
+
+<p>The <tt>Knox SSO Public Key</tt> Ambari property should be set to the output of that command. In the same section, set the <tt>Knox Enabled</tt> setting to “ON”.</p>
+<p>After these changes have been made, the following Metron components must be restarted:</p>
+<ul>
+
+<li>Metron Client</li>
+<li>Metron Alerts UI</li>
+<li>Metron Management UI</li>
+<li>Metron REST</li>
+</ul>
+<p>Any change to the settings described in this section will cause Ambari to suggest restarting these services. Either restart them manually or follow the prompt in Ambari.</p>
+<p>Note: Knox can only be enabled for Metron when Metron authentication is set to <tt>LDAP</tt> (the <tt>LDAP Enabled</tt> setting in Ambari).<br />
+If <tt>LDAP</tt> is not enabled an error will be thrown when any of the previous components are restarted.</p></div>
+<div class="section">
+<h3><a name="Update_Quicklinks_.28Optional.29"></a>Update Quicklinks (Optional)</h3>
+<p>Currently there is no way to dynamicly change the quick links in Ambari to the new Knox urls so it must be done manually. Locate this file on the Ambari server host:</p>
+
+<div>
+<div>
+<pre class="source">/var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-$METRON_MPACK_VERSION/common-services/METRON/$METRON_VERSION/quicklinks/quicklinks.json
+</pre></div></div>
+
+<p>Quicklinks for each component are defined as an array of json objects under the <tt>/configuration/links/</tt> attribute.</p>
+<p>Locate the json object for the component you want to update. Change the <tt>url</tt> attribute to match the new Knox url. For example, to update the Metron REST quick link, locate the json object with the <tt>label</tt> property set to <tt>Swagger UI</tt>. Change the <tt>url</tt> property from <tt>%@://%@:%@/swagger-ui.html</tt> to <tt>https://<knox gateway host>:<knox gateway port>/gateway/metron/metron-rest/swagger-ui.html</tt>.</p>
+<p>The json object should look like this in full dev:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "name": "metron_rest_ui",
+ "label": "Swagger UI",
+ "requires_user_name": "false",
+ "component_name": "METRON_REST",
+ "url":"https://node1:8443/gateway/metron/metron-rest/swagger-ui.html",
+ "port":{
+ "http_property": "metron_rest_port",
+ "http_default_port": "8082",
+ "https_property": "metron_rest_port",
+ "https_default_port": "8082",
+ "regex": "^(\\d+)$",
+ "site": "metron-rest-env"
+ }
+}
+</pre></div></div>
+
+<p>Repeat for the Alerts UI and Management UI. Any update to this file requires an Ambari server restart.</p></div></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-interface/metron-alerts/index.html b/current-book/metron-interface/metron-alerts/index.html
index a150adb..d4f7a08 100644
--- a/current-book/metron-interface/metron-alerts/index.html
+++ b/current-book/metron-interface/metron-alerts/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-interface/metron-alerts/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-interface/metron-alerts/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – </title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active "></li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,13 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-down"></span>Interface</a>
+ <ul class="nav nav-list">
<li class="active"><a href="#"><span class="none"></span>Alerts</a></li>
<li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
<li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ </ul>
+</li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -95,6 +99,7 @@
<li><a href="#Prerequisites">Prerequisites</a></li>
<li><a href="#Development_Setup">Development Setup</a></li>
<li><a href="#E2E_Tests">E2E Tests</a></li>
+<li><a href="#Cypress_Tests">Cypress Tests</a></li>
<li><a href="#Mpack_Integration">Mpack Integration</a></li>
<li><a href="#Installing_on_an_existing_Cluster">Installing on an existing Cluster</a></li>
</ul>
@@ -297,6 +302,52 @@
</li>
</ol>
<p><b>NOTE</b>: <i>e2e tests cover all the general workflows and we will extend them as we need</i></p></div></div>
+<div class="section">
+<h2><a name="Cypress_Tests"></a>Cypress Tests</h2>
+<p>Cypress is a test framework that allows developers and test engineers to create E2E or Integration tests and run them inside the browser. It differs from other testing tools by choosing to not use selenium webdriver to control the browser.</p>
+<p>Cypress is added to our project as a developer dependency. This means it’s installed when you run:</p>
+
+<div>
+<div>
+<pre class="source">npm ci
+</pre></div></div>
+
+<p>or</p>
+
+<div>
+<div>
+<pre class="source">npm install
+</pre></div></div>
+
+<p>Currently both Protractor and Cypress tests are available, so the previous section of this text is still relevant. However, in the near future we’re planning to migrate fully to Cypress. You can find the public discussion about this in the following link: <a class="externalLink" href="https://lists.apache.org/thread.html/b6a0272c7809c05e8b7aff20171720e8ec76f8a0e9481169c37a4a4a@%3Cdev.metron.apache.org%3E">Discussion thread</a></p>
+<div class="section">
+<h3><a name="Steps_to_run"></a>Steps to run</h3>
+<p>The easiest way to run Cypress locally is with the following command:</p>
+
+<div>
+<div>
+<pre class="source">npm run cypress:ci
+</pre></div></div>
+
+<p>The same command runs on Travis CI as part of our build process.</p>
+<p>If you would like to get some additional information or run test suites one by one you could start the test server manually</p>
+
+<div>
+<div>
+<pre class="source">npm run build && npm run start:ci
+</pre></div></div>
+
+<p>and then you can reach the Cypress Dashboard locally with the following command</p>
+
+<div>
+<div>
+<pre class="source">npm run cypress:open
+</pre></div></div>
+
+<p>From the dashboard, you’ll be able to run tests separately and reach additional information about the test runs.</p></div>
+<div class="section">
+<h3><a name="Learn_more"></a>Learn more</h3>
+<p>If you like to learn more about Cypress based tests please visit <a class="externalLink" href="http://cypress.io">Cypress.io</a>. You can find more information about debuggin in this <a class="externalLink" href="https://docs.cypress.io/guides/guides/debugging.html#Using-debugger">section of the official documentation</a>.</p></div></div>
</div>
</div>
</div>
diff --git a/current-book/metron-interface/metron-config/index.html b/current-book/metron-interface/metron-config/index.html
index 6b781f4..b9f1dd8 100644
--- a/current-book/metron-interface/metron-config/index.html
+++ b/current-book/metron-interface/metron-config/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-interface/metron-config/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-interface/metron-config/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Management UI</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Management UI</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,13 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-down"></span>Interface</a>
+ <ul class="nav nav-list">
<li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
<li class="active"><a href="#"><span class="none"></span>Config</a></li>
<li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ </ul>
+</li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-interface/metron-rest/index.html b/current-book/metron-interface/metron-rest/index.html
index 54a8861..5700f71 100644
--- a/current-book/metron-interface/metron-rest/index.html
+++ b/current-book/metron-interface/metron-rest/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-interface/metron-rest/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-interface/metron-rest/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron REST</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron REST</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,13 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-down"></span>Interface</a>
+ <ul class="nav nav-list">
<li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
<li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
<li class="active"><a href="#"><span class="none"></span>Rest</a></li>
+ </ul>
+</li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -92,6 +96,16 @@
<h1>Metron REST</h1>
<p><a name="Metron_REST"></a></p>
<p>This module provides a RESTful API for interacting with Metron.</p>
+<ul>
+
+<li><a href="#Prerequisites">Prerequisites</a></li>
+<li><a href="#Installation">Installation</a></li>
+<li><a href="#Configuration">Configuration</a></li>
+<li><a href="#Usage">Usage</a></li>
+<li><a href="#Security">Security</a></li>
+<li><a href="#API">API</a></li>
+<li><a href="#Testing">Testing</a></li>
+</ul>
<div class="section">
<h2><a name="Prerequisites"></a>Prerequisites</h2>
<ul>
@@ -169,18 +183,6 @@
</thead><tbody>
<tr class="b">
-<td> METRON_JDBC_DRIVER </td>
-<td> JDBC driver class</td></tr>
-<tr class="a">
-<td> METRON_JDBC_URL </td>
-<td> JDBC url</td></tr>
-<tr class="b">
-<td> METRON_JDBC_USERNAME </td>
-<td> JDBC username</td></tr>
-<tr class="a">
-<td> METRON_JDBC_PLATFORM </td>
-<td> JDBC platform (one of h2, mysql, postgres, oracle</td></tr>
-<tr class="b">
<td> ZOOKEEPER </td>
<td> Zookeeper quorum (ex. node1:2181,node2:2181)</td></tr>
<tr class="a">
@@ -198,46 +200,56 @@
<tr class="a">
<th> Environment Variable </th>
-<th> Description </th>
+<th> Description </th>
<th> Required </th>
<th> Default</th></tr>
</thead><tbody>
<tr class="b">
<td> METRON_LOG_DIR </td>
-<td> Directory where the log file is written </td>
+<td> Directory where the log file is written </td>
<td> Optional </td>
<td> /var/log/metron/</td></tr>
<tr class="a">
<td> METRON_PID_FILE </td>
-<td> File where the pid is written </td>
+<td> File where the pid is written </td>
<td> Optional </td>
<td> /var/run/metron/</td></tr>
<tr class="b">
<td> METRON_REST_PORT </td>
-<td> REST application port </td>
+<td> REST application port </td>
<td> Optional </td>
<td> 8082</td></tr>
<tr class="a">
<td> METRON_JDBC_CLIENT_PATH </td>
-<td> Path to JDBC client jar </td>
+<td> Path to JDBC client jar </td>
<td> Optional </td>
<td> H2 is bundled</td></tr>
<tr class="b">
<td> METRON_TEMP_GROK_PATH </td>
-<td> Temporary directory used to test grok statements </td>
+<td> Temporary directory used to test grok statements </td>
<td> Optional </td>
<td> ./patterns/temp</td></tr>
<tr class="a">
<td> METRON_DEFAULT_GROK_PATH </td>
-<td> Defaults HDFS directory used to store grok statements </td>
+<td> Defaults HDFS directory used to store grok statements </td>
<td> Optional </td>
<td> /apps/metron/patterns</td></tr>
<tr class="b">
<td> SECURITY_ENABLED </td>
-<td> Enables Kerberos support </td>
+<td> Enables Kerberos support </td>
<td> Optional </td>
<td> false</td></tr>
+<tr class="a">
+<td> METRON_USER_ROLE </td>
+<td> Name of the role at the authentication provider that provides user access to Metron. </td>
+<td> Optional </td>
+<td> USER</td></tr>
+<tr class="b">
+<td> METRON_ADMIN_ROLE </td>
+<td> Name of the role at the authentication provider that provides administrative access to Metron.</td>
+<td> Optional </td>
+<td> ADMIN</td></tr>
</tbody>
</table></div>
<div class="section">
@@ -252,6 +264,22 @@
</thead><tbody>
<tr class="b">
+<td> METRON_JDBC_DRIVER </td>
+<td> JDBC driver class </td>
+<td> Optional</td></tr>
+<tr class="a">
+<td> METRON_JDBC_URL </td>
+<td> JDBC url </td>
+<td> Optional</td></tr>
+<tr class="b">
+<td> METRON_JDBC_USERNAME </td>
+<td> JDBC username </td>
+<td> Optional</td></tr>
+<tr class="a">
+<td> METRON_JDBC_PLATFORM </td>
+<td> JDBC platform (one of h2, mysql, postgres, oracle) </td>
+<td> Optional</td></tr>
+<tr class="b">
<td> METRON_JVMFLAGS </td>
<td> JVM flags added to the start command </td>
<td> Optional</td></tr>
@@ -275,9 +303,190 @@
</table>
<p>These are set in the <tt>/etc/default/metron</tt> file.</p></div></div>
<div class="section">
-<h2><a name="Database_setup"></a>Database setup</h2>
+<h2><a name="Usage"></a>Usage</h2>
+<p>The REST application can be accessed with the Swagger UI at <a class="externalLink" href="http://host:port/swagger-ui.html#/">http://host:port/swagger-ui.html#/</a>. The default port is 8082.</p>
+<div class="section">
+<h3><a name="Logging"></a>Logging</h3>
+<p>Logging for the REST application can be configured in Ambari. Log levels can be changed at the root, package and class level:</p>
+<ol style="list-style-type: decimal">
+
+<li>
+
+<p>Navigate to Services > Metron > Configs > REST and locate the <tt>Metron Spring options</tt> setting.</p>
+</li>
+<li>
+
+<p>Logging configuration is exposed through Spring properties as explained <a class="externalLink" href="https://docs.spring.io/spring-boot/docs/current/reference/html/howto-logging.html#howto-logging">here</a>.</p>
+</li>
+<li>
+
+<p>The root logging level defaults to ERROR but can be changed to INFO by adding <tt>--logging.level.root=INFO</tt> to the <tt>Metron Spring options</tt> setting.</p>
+</li>
+<li>
+
+<p>The Metron REST logging level can be changed to INFO by adding <tt>--logging.level.org.apache.metron.rest=INFO</tt>.</p>
+</li>
+<li>
+
+<p>HTTP request and response logging can be enabled by adding <tt>--logging.level.org.springframework.web.filter.CommonsRequestLoggingFilter=DEBUG --logging.level.org.apache.metron.rest.web.filter.ResponseLoggingFilter=DEBUG</tt>.</p>
+</li>
+</ol></div>
+<div class="section">
+<h3><a name="Spring_Profiles"></a>Spring Profiles</h3>
+<p>The REST application comes with a few <a class="externalLink" href="http://docs.spring.io/autorepo/docs/spring-boot/current/reference/html/boot-features-profiles.html">Spring Profiles</a> to aid in testing and development.</p>
+<table border="0" class="table table-striped">
+<thead>
+
+<tr class="a">
+<th> Profile </th>
+<th> Description </th></tr>
+</thead><tbody>
+
+<tr class="b">
+<td> test </td>
+<td> adds test users <tt>[user, user1, user2, admin]</tt> to the database with password “<tt>password</tt>”. sets variables to in-memory services, only used for integration testing </td></tr>
+<tr class="a">
+<td> dev </td>
+<td> adds test users <tt>[user, user1, user2, admin]</tt> to the database with password “<tt>password</tt>” </td></tr>
+<tr class="b">
+<td> vagrant </td>
+<td> sets configuration variables to match the Metron vagrant environment </td></tr>
+<tr class="a">
+<td> docker </td>
+<td> sets configuration variables to match the Metron docker environment </td></tr>
+</tbody>
+</table>
+<p>Setting active profiles is done with the METRON_SPRING_PROFILES_ACTIVE variable. For example, set this variable in <tt>/etc/default/metron</tt> to configure the REST application for the Vagrant environment and add a test user:</p>
+
+<div>
+<div>
+<pre class="source">METRON_SPRING_PROFILES_ACTIVE="vagrant,dev"
+</pre></div></div>
+</div></div>
+<div class="section">
+<h2><a name="Security"></a>Security</h2>
+<ul>
+
+<li><a href="#Kerberos">Kerberos</a></li>
+<li><a href="#LDAP_Authentication">LDAP Authentication</a></li>
+<li><a href="#JDBC_Authentication">JDBC Authentication</a></li>
+</ul>
+<div class="section">
+<h3><a name="Kerberos"></a>Kerberos</h3>
+<p>Metron REST can be configured for a cluster with Kerberos enabled. A client JAAS file is required for Kafka and Zookeeper and a Kerberos keytab for the metron user principal is required for all other services. Configure these settings in the <tt>/etc/default/metron</tt> file:</p>
+
+<div>
+<div>
+<pre class="source">SECURITY_ENABLED=true
+METRON_JVMFLAGS="-Djava.security.auth.login.config=$METRON_HOME/client_jaas.conf"
+METRON_PRINCIPAL_NAME="metron@EXAMPLE.COM"
+METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab"
+</pre></div></div>
+</div>
+<div class="section">
+<h3><a name="LDAP_Authentication"></a>LDAP Authentication</h3>
+<p>Metron REST can be configured to use LDAP for authentication and roles. Use the following steps to enable LDAP.</p>
+<ol style="list-style-type: decimal">
+
+<li>
+
+<p>In Ambari, go to Metron > Config > Security > Roles</p>
+<ul>
+
+<li>
+
+<p>Set “User Role Name” to the name of the role at the authentication provider that provides user level access to Metron.</p>
+</li>
+<li>
+
+<p>Set “Admin Role Name” to the name of the role at the authentication provider that provides administrative access to Metron.</p>
+</li>
+</ul>
+</li>
+<li>
+
+<p>In Ambari, go to Metron > Config > Security > LDAP</p>
+<ul>
+
+<li>
+
+<p>Turn on LDAP using the toggle.</p>
+</li>
+<li>
+
+<p>Set “LDAP URL” to your LDAP instance. For example, <tt>ldap://<host>:<port></tt>.</p>
+</li>
+<li>
+
+<p>Set “Bind User” to the name of the bind user. For example, <tt>cn=admin,dc=apache,dc=org</tt>.</p>
+</li>
+<li>
+
+<p>Set the “Bind User Password”</p>
+</li>
+<li>
+
+<p>Other fields may be required depending on your LDAP configuration.</p>
+</li>
+</ul>
+</li>
+<li>
+
+<p>Save the changes and restart the required services.</p>
+</li>
+</ol>
+<p>By default, configuration will default to matching Knox’s Demo LDAP for convenience. This should only be used for development purposes. Manual instructions for setting up demo LDAP and finalizing configuration (e.g. setting up the user LDIF file) can be found in the <a href="../../metron-deployment/development/index.html#knox-demo-ldap">Development README</a>.</p>
+<div class="section">
+<h4><a name="LDAPS"></a>LDAPS</h4>
+<p>There is configuration to provide a path to a truststore with SSL certificates and provide a password. Users should import certificates as needed to appropriate truststores. An example of doing this is:</p>
+
+<div>
+<div>
+<pre class="source">keytool -import -alias <alias> -file <certificate> -keystore <keystore_file> -storepass <password>
+</pre></div></div>
+</div>
+<div class="section">
+<h4><a name="Roles"></a>Roles</h4>
+<p>Roles used by Metron are <tt>ROLE_ADMIN</tt> and <tt>ROLE_USER</tt>. Metron will use a property in a group containing the appropriate role to construct this.</p>
+<p>Metron can be configured to map the roles defined in your authorization provider to the authorities used internally for access control. This can be configured under Security > Roles in Ambari.</p>
+<p>For example, our ldif file could create this group:</p>
+
+<div>
+<div>
+<pre class="source">dn: cn=admin,ou=groups,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass: groupofnames
+cn: admin
+description:admin group
+member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
+</pre></div></div>
+
+<p>If we are using “cn” as our role attribute, Metron will give the “admin” user the role “ROLE_ADMIN”.</p>
+<p>Similarly, we could give a user “sam” ROLE_USER with the following group:</p>
+
+<div>
+<div>
+<pre class="source">dn: cn=user,ou=groups,dc=hadoop,dc=apache,dc=org
+objectclass:top
+objectclass: groupofnames
+cn: user
+description: user group
+member: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org
+</pre></div></div>
+</div></div>
+<div class="section">
+<h3><a name="JDBC_Authentication"></a>JDBC Authentication</h3>
<p>The REST application persists data in a relational database and requires a dedicated database user and database (see <a class="externalLink" href="https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html">https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html</a> for more detail).<br />
Spring uses Hibernate as the default ORM framework but another framework is needed becaused Hibernate is not compatible with the Apache 2 license. For this reason Metron uses <a class="externalLink" href="https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html#boot-features-embedded-database-support">EclipseLink</a>. See the <a class="externalLink" href="https://github.com/spring-projects/spring-data-examples/tree/master/jpa/eclipselink">Spring Data JPA - EclipseLink</a> project for an example on how to configure EclipseLink in Spring.</p>
+<p>The metron-rest module uses <a class="externalLink" href="http://projects.spring.io/spring-security/">Spring Security</a> for authentication and stores user credentials in the relational database configured above. The required tables are created automatically the first time the application is started so that should be done first. For example (continuing the MySQL example above), users can be added by connecting to MySQL and running:</p>
+
+<div>
+<div>
+<pre class="source">use metronrest;
+insert into users (username, password, enabled) values ('your_username','your_password',1);
+insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
+</pre></div></div>
+</div>
<div class="section">
<h3><a name="Development"></a>Development</h3>
<p>The REST application comes with <a class="externalLink" href="https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html#boot-features-embedded-database-support">embedded database support</a> for development purposes.</p>
@@ -370,107 +579,6 @@
</li>
</ol></div></div></div>
<div class="section">
-<h2><a name="Usage"></a>Usage</h2>
-<p>The REST application can be accessed with the Swagger UI at <a class="externalLink" href="http://host:port/swagger-ui.html#/">http://host:port/swagger-ui.html#/</a>. The default port is 8082.</p></div>
-<div class="section">
-<h2><a name="Security"></a>Security</h2>
-<div class="section">
-<h3><a name="Authentication"></a>Authentication</h3>
-<p>The metron-rest module uses <a class="externalLink" href="http://projects.spring.io/spring-security/">Spring Security</a> for authentication and stores user credentials in the relational database configured above. The required tables are created automatically the first time the application is started so that should be done first. For example (continuing the MySQL example above), users can be added by connecting to MySQL and running:</p>
-
-<div>
-<div>
-<pre class="source">use metronrest;
-insert into users (username, password, enabled) values ('your_username','your_password',1);
-insert into authorities (username, authority) values ('your_username', 'ROLE_USER');
-</pre></div></div>
-</div>
-<div class="section">
-<h3><a name="Kerberos"></a>Kerberos</h3>
-<p>Metron REST can be configured for a cluster with Kerberos enabled. A client JAAS file is required for Kafka and Zookeeper and a Kerberos keytab for the metron user principal is required for all other services. Configure these settings in the <tt>/etc/default/metron</tt> file:</p>
-
-<div>
-<div>
-<pre class="source">SECURITY_ENABLED=true
-METRON_JVMFLAGS="-Djava.security.auth.login.config=$METRON_HOME/client_jaas.conf"
-METRON_PRINCIPAL_NAME="metron@EXAMPLE.COM"
-METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab"
-</pre></div></div>
-</div>
-<div class="section">
-<h3><a name="LDAP"></a>LDAP</h3>
-<p>Metron REST can be configured to use LDAP for authentication and roles. Configuration can be performed via Ambari in the “Security” tab.</p>
-<p>Configuration will default to matching Knox’s Demo LDAP for convenience. This should only be used for development purposes. Manual instructions for setting up demo LDAP and finalizing configuration (e.g. setting up the user LDIF file) can be found in the <a href="../../metron-deployment/development/index.html#knox-demo-ldap">Development README</a>.</p>
-<div class="section">
-<h4><a name="LDAPS"></a>LDAPS</h4>
-<p>There is configuration to provide a path to a truststore with SSL certificates and provide a password. Users should import certificates as needed to appropriate truststores. An example of doing this is:</p>
-
-<div>
-<div>
-<pre class="source">keytool -import -alias <alias> -file <certificate> -keystore <keystore_file> -storepass <password>
-</pre></div></div>
-</div>
-<div class="section">
-<h4><a name="Roles"></a>Roles</h4>
-<p>Roles used by Metron are ROLE_ADMIN and ROLE_USER. Metron will use a property in a group containing the appropriate role to construct this.</p>
-<p>For example, our ldif file could create this group:</p>
-
-<div>
-<div>
-<pre class="source">dn: cn=admin,ou=groups,dc=hadoop,dc=apache,dc=org
-objectclass:top
-objectclass: groupofnames
-cn: admin
-description:admin group
-member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
-</pre></div></div>
-
-<p>If we are using “cn” as our role attribute, Metron will give the “admin” user the role “ROLE_ADMIN”.</p>
-<p>Similarly, we could give a user “sam” ROLE_USER with the following group:</p>
-
-<div>
-<div>
-<pre class="source">dn: cn=user,ou=groups,dc=hadoop,dc=apache,dc=org
-objectclass:top
-objectclass: groupofnames
-cn: user
-description: user group
-member: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org
-</pre></div></div>
-</div></div></div>
-<div class="section">
-<h2><a name="Spring_Profiles"></a>Spring Profiles</h2>
-<p>The REST application comes with a few <a class="externalLink" href="http://docs.spring.io/autorepo/docs/spring-boot/current/reference/html/boot-features-profiles.html">Spring Profiles</a> to aid in testing and development.</p>
-<table border="0" class="table table-striped">
-<thead>
-
-<tr class="a">
-<th> Profile </th>
-<th> Description </th></tr>
-</thead><tbody>
-
-<tr class="b">
-<td> test </td>
-<td> adds test users <tt>[user, user1, user2, admin]</tt> to the database with password “<tt>password</tt>”. sets variables to in-memory services, only used for integration testing </td></tr>
-<tr class="a">
-<td> dev </td>
-<td> adds test users <tt>[user, user1, user2, admin]</tt> to the database with password “<tt>password</tt>” </td></tr>
-<tr class="b">
-<td> vagrant </td>
-<td> sets configuration variables to match the Metron vagrant environment </td></tr>
-<tr class="a">
-<td> docker </td>
-<td> sets configuration variables to match the Metron docker environment </td></tr>
-</tbody>
-</table>
-<p>Setting active profiles is done with the METRON_SPRING_PROFILES_ACTIVE variable. For example, set this variable in <tt>/etc/default/metron</tt> to configure the REST application for the Vagrant environment and add a test user:</p>
-
-<div>
-<div>
-<pre class="source">METRON_SPRING_PROFILES_ACTIVE="vagrant,dev"
-</pre></div></div>
-</div>
-<div class="section">
<h2><a name="Pcap_Query"></a>Pcap Query</h2>
<p>The REST application exposes endpoints for querying Pcap data. For more information about filtering options see <a href="../../metron-platform/metron-pcap-backend/index.html#Query_Filter_Utility">Query Filter Utility</a>.</p>
<p>There is an endpoint available that will return Pcap data in <a class="externalLink" href="https://wiki.wireshark.org/PDML">PDML</a> format. <a class="externalLink" href="https://www.wireshark.org/">Wireshark</a> must be installed for this feature to work. Installing wireshark in CentOS can be done with <tt>yum -y install wireshark</tt>.</p>
@@ -610,6 +718,14 @@
<tr class="a">
<td> <a href="#GET_apiv1sensorparserconfigname"> <tt>GET /api/v1/sensor/parser/config/{name}</tt></a></td></tr>
<tr class="b">
+<td> <a href="#POST_apiv1sensorparsergroup"> <tt>POST /api/v1/sensor/parser/group</tt></a></td></tr>
+<tr class="a">
+<td> <a href="#GET_apiv1sensorparsergroupname"> <tt>GET /api/v1/sensor/parser/group/{name}</tt></a></td></tr>
+<tr class="b">
+<td> <a href="#GET_apiv1sensorparsergroup"> <tt>GET /api/v1/sensor/parser/group</tt></a></td></tr>
+<tr class="a">
+<td> <a href="#DELETE_apiv1sensorparsergroupname"> <tt>DELETE /api/v1/sensor/parser/group/{name}</tt></a></td></tr>
+<tr class="b">
<td> <a href="#POST_apiv1stellarapplytransformations"> <tt>POST /api/v1/stellar/apply/transformations</tt></a></td></tr>
<tr class="a">
<td> <a href="#GET_apiv1stellarlist"> <tt>GET /api/v1/stellar/list</tt></a></td></tr>
@@ -1397,7 +1513,7 @@
<li>Returns:
<ul>
-<li>200 - Returns a list of available enrichments</li>
+<li>200 - Returns a list of available enrichments in lexicographical order</li>
</ul>
</li>
</ul></div>
@@ -1654,6 +1770,75 @@
</li>
</ul></div>
<div class="section">
+<h3><a name="POST_.2Fapi.2Fv1.2Fsensor.2Fparser.2Fgroup"></a><tt>POST /api/v1/sensor/parser/group</tt></h3>
+<ul>
+
+<li>Description: Updates or creates a SensorParserGroup in Zookeeper</li>
+<li>Input:
+<ul>
+
+<li>sensorParserGroup - SensorParserGroup</li>
+</ul>
+</li>
+<li>Returns:
+<ul>
+
+<li>200 - SensorParserGroup updated. Returns saved SensorParserGroup</li>
+<li>201 - SensorParserGroup created. Returns saved SensorParserGroup</li>
+</ul>
+</li>
+</ul></div>
+<div class="section">
+<h3><a name="GET_.2Fapi.2Fv1.2Fsensor.2Fparser.2Fgroup.2F.7Bname.7D"></a><tt>GET /api/v1/sensor/parser/group/{name}</tt></h3>
+<ul>
+
+<li>Description: Retrieves a SensorParserGroup from Zookeeper</li>
+<li>Input:
+<ul>
+
+<li>name - SensorParserGroup name</li>
+</ul>
+</li>
+<li>Returns:
+<ul>
+
+<li>200 - Returns SensorParserGroup</li>
+<li>404 - SensorParserGroup is missing</li>
+</ul>
+</li>
+</ul></div>
+<div class="section">
+<h3><a name="GET_.2Fapi.2Fv1.2Fsensor.2Fparser.2Fgroup"></a><tt>GET /api/v1/sensor/parser/group</tt></h3>
+<ul>
+
+<li>Description: Retrieves all SensorParserGroups from Zookeeper</li>
+<li>Returns:
+<ul>
+
+<li>200 - Returns all SensorParserGroups</li>
+</ul>
+</li>
+</ul></div>
+<div class="section">
+<h3><a name="DELETE_.2Fapi.2Fv1.2Fsensor.2Fparser.2Fgroup.2F.7Bname.7D"></a><tt>DELETE /api/v1/sensor/parser/group/{name}</tt></h3>
+<ul>
+
+<li>Description: Deletes a SensorParserGroup from Zookeeper</li>
+<li>Input:
+<ul>
+
+<li>name - SensorParserGroup name</li>
+</ul>
+</li>
+<li>Returns:
+<ul>
+
+<li>200 - SensorParserGroup was deleted</li>
+<li>404 - SensorParserGroup is missing</li>
+</ul>
+</li>
+</ul></div>
+<div class="section">
<h3><a name="POST_.2Fapi.2Fv1.2Fstellar.2Fapply.2Ftransformations"></a><tt>POST /api/v1/stellar/apply/transformations</tt></h3>
<ul>
diff --git a/current-book/metron-platform/Performance-tuning-guide.html b/current-book/metron-platform/Performance-tuning-guide.html
index 42461be..facc2bf 100644
--- a/current-book/metron-platform/Performance-tuning-guide.html
+++ b/current-book/metron-platform/Performance-tuning-guide.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/Performance-tuning-guide.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/Performance-tuning-guide.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Performance Tuning Guide</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Performance Tuning Guide</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li class="active"><a href="#"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -59,10 +57,11 @@
<li><a href="../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
<li><a href="../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
<li><a href="../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li><a href="../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
<li><a href="../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
<li><a href="../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
- <li><a href="../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li><a href="../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
<li><a href="../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
<li><a href="../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
@@ -137,7 +136,7 @@
<p>worked well in nearly all cases. As a general rule, it was optimal to set spout parallelism equal to the number of partitions used in your Kafka topic. Any greater parallelism will leave you with idle consumers since Kafka limits the max number of consumers to the number of partitions. This is important because Kafka has certain ordering guarantees for message delivery per partition that would not be possible if more than one consumer in a given consumer group were able to read from that partition.</p></div></div>
<div class="section">
<h2><a name="Sensor_Topology_Tuning_Suggestions"></a>Sensor Topology Tuning Suggestions</h2>
-<p>If you are using stellar field transformations in your sensors, by default, stellar expressions are not cached. Sensors that use stellar field transformations by see a performance boost by turning on caching via setting the <tt>cacheConfig</tt> <a href="metron-parsers/index.html#parser_configuration">property</a>. This is beneficial if your transformations:</p>
+<p>If you are using stellar field transformations in your sensors, by default, stellar expressions are not cached. Sensors that use stellar field transformations by see a performance boost by turning on caching via setting the <tt>cacheConfig</tt> <a href="metron-parsers-common/index.html#parser_configuration">property</a>. This is beneficial if your transformations:</p>
<ul>
<li>Are complex (e.g. <tt>ENRICHMENT_GET</tt> calls or other high latency calls)</li>
@@ -258,7 +257,7 @@
</ul>
<p><b>Parsers</b></p>
<p>This is a mapping of the various performance tuning properties for parsers and how they are materialized.</p>
-<p>See more detail on starting parsers <a class="externalLink" href="https://github.com/apache/metron/blob/master/metron-platform/metron-parsers/README.md#starting-the-parser-topology">here</a></p>
+<p>See more detail on starting parsers <a class="externalLink" href="https://github.com/apache/metron/blob/master/metron-platform/metron-parsing/metron-parsers-common/README.md#starting-the-parser-topology">here</a></p>
<table border="0" class="table table-striped">
<thead>
@@ -783,7 +782,7 @@
<div>
<div>
-<pre class="source">/usr/metron/0.7.0/bin/start_parser_topology.sh \
+<pre class="source">$METRON_HOME/bin/start_parser_topology.sh \
-e ~metron/.storm/storm-bro.config \
-esc ~/.storm/spout-bro.config \
-k $BROKERLIST \
@@ -798,7 +797,7 @@
-z $ZOOKEEPER \
</pre></div></div>
-<p>From the usage docs, here are the options we’ve used. The full reference can be found <a href="../metron-platform/metron-parsers/index.html#Starting_the_Parser_Topology">here</a>.</p>
+<p>From the usage docs, here are the options we’ve used. The full reference can be found <a href="../metron-platform/metron-parsing/metron-parsers-common/index.html#Starting_the_Parser_Topology">here</a>.</p>
<div>
<div>
@@ -969,7 +968,7 @@
export STORM_UI=http://node1:8744
export ELASTIC=http://node1:9200
export ZOOKEEPER=node1:2181
-export METRON_VERSION=0.7.0
+export METRON_VERSION=0.7.1
export METRON_HOME=/usr/metron/${METRON_VERSION}
</pre></div></div>
diff --git a/current-book/metron-platform/index.html b/current-book/metron-platform/index.html
index 11d0324..9c523f9 100644
--- a/current-book/metron-platform/index.html
+++ b/current-book/metron-platform/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Current Build</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Current Build</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li class="active"><a href="#"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li><a href="../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -59,10 +57,11 @@
<li><a href="../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
<li><a href="../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
<li><a href="../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li><a href="../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
<li><a href="../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
<li><a href="../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
- <li><a href="../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li><a href="../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
<li><a href="../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
<li><a href="../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
@@ -105,14 +104,14 @@
-->
<h1>Current Build</h1>
<p><a name="Current_Build"></a></p>
-<p>The latest build of metron-platform is 0.7.0.</p>
+<p>The latest build of metron-platform is 0.7.1.</p>
<p>We are still in the process of merging/porting additional features from our production code base into this open source release. This release will be followed by a number of additional beta releases until the port is complete. We will also work on getting additional documentation and user/developer guides to the community as soon as we can. At this time we offer no support for the beta software, but will try to respond to requests as promptly as we can.</p>
<p><a name="metron-platform"></a></p>
<h1>metron-platform</h1>
<p>Extensible set of Storm topologies and topology attributes for streaming, enriching, indexing, and storing telemetry in Hadoop. General information on Metron is available at <a class="externalLink" href="https://metron.apache.org/">https://metron.apache.org/</a></p>
<p><a name="Documentation"></a></p>
<h1>Documentation</h1>
-<p>Please see documentation within each individual module for description and usage instructions. Sample topologies are provided under Metron_Topologies to get you started with the framework. We pre-assume knowledge of Hadoop, Storm, Kafka, and HBase.</p>
+<p>Please see documentation within each individual module for description and usage instructions. Sample topologies are provided under Metron_Topologies to get you started with the framework. We pre-assume knowledge of Hadoop, Storm, Kafka, Zookeeper, and HBase.</p>
</div>
</div>
</div>
diff --git a/current-book/metron-platform/metron-common/index.html b/current-book/metron-platform/metron-common/index.html
index baadb3f..362712f 100644
--- a/current-book/metron-platform/metron-common/index.html
+++ b/current-book/metron-platform/metron-common/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-common/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-common/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Contents</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Contents</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -59,10 +57,11 @@
<li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
<li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
<li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
<li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
<li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
- <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
<li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
<li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
@@ -109,6 +108,7 @@
<ul>
<li><a href="#Stellar_Language">Stellar Language</a></li>
+<li><a href="#High_Level_Architecture">High Level Architecture</a></li>
<li><a href="#Global_Configuration">Global Configuration</a></li>
<li><a href="#Validation_Framework">Validation Framework</a></li>
<li><a href="#Management_Utility">Management Utility</a></li>
@@ -170,127 +170,215 @@
<thead>
<tr class="a">
-<th> Property Name </th>
+<th> Property Name </th>
<th> Subsystem </th>
<th> Type </th>
-<th> Ambari Property </th></tr>
+<th> Ambari Property </th></tr>
</thead><tbody>
<tr class="b">
-<td> <a href="../metron-elasticsearch/index.html#es.clustername"><tt>es.clustername</tt></a> </td>
+<td> <a href="../metron-elasticsearch/index.html#es.clustername"><tt>es.clustername</tt></a> </td>
<td> Indexing </td>
<td> String </td>
-<td> <tt>es_cluster_name</tt> </td></tr>
+<td> <tt>es_cluster_name</tt> </td></tr>
<tr class="a">
-<td> <a href="../metron-elasticsearch/index.html#es.ip"><tt>es.ip</tt></a> </td>
+<td> <a href="../metron-elasticsearch/index.html#es.ip"><tt>es.ip</tt></a> </td>
<td> Indexing </td>
<td> String </td>
-<td> <tt>es_hosts</tt> </td></tr>
+<td> <tt>es_hosts</tt> & <tt>es_port</tt> </td></tr>
<tr class="b">
-<td> <a href="../metron-elasticsearch/index.html#es.port"><tt>es.port</tt></a> </td>
+<td> <a href="../metron-elasticsearch/index.html#es.port"><tt>es.port</tt></a> </td>
<td> Indexing </td>
<td> String </td>
-<td> <tt>es_port</tt> </td></tr>
+<td> N/A </td></tr>
<tr class="a">
-<td> <a href="../metron-elasticsearch/index.html#es.date.format"><tt>es.date.format</tt></a> </td>
+<td> <a href="../metron-elasticsearch/index.html#es.date.format"><tt>es.date.format</tt></a> </td>
<td> Indexing </td>
<td> String </td>
-<td> <tt>es_date_format</tt> </td></tr>
+<td> <tt>es_date_format</tt> </td></tr>
<tr class="b">
-<td> <a href="../metron-elasticsearch/index.html#es.client.settings"><tt>es.client.settings</tt></a> </td>
+<td> <a href="../metron-elasticsearch/index.html#es.client.settings"><tt>es.client.settings</tt></a> </td>
<td> Indexing </td>
<td> Object </td>
-<td> N/A </td></tr>
+<td> N/A </td></tr>
<tr class="a">
-<td> <a href="#validation-framework"><tt>fieldValidations</tt></a> </td>
+<td> <a href="../metron-solr/index.html#configuration"><tt>solr.zookeeper</tt></a> </td>
+<td> Indexing </td>
+<td> String </td>
+<td> <tt>solr_zookeeper_url</tt> </td></tr>
+<tr class="b">
+<td> <a href="../metron-solr/index.html#configuration"><tt>solr.commitPerBatch</tt></a> </td>
+<td> Indexing </td>
+<td> String </td>
+<td> N/A </td></tr>
+<tr class="a">
+<td> <a href="../metron-solr/index.html#configuration"><tt>solr.commit.soft</tt></a> </td>
+<td> Indexing </td>
+<td> String </td>
+<td> N/A </td></tr>
+<tr class="b">
+<td> <a href="../metron-solr/index.html#configuration"><tt>solr.commit.waitSearcher</tt></a> </td>
+<td> Indexing </td>
+<td> String </td>
+<td> N/A </td></tr>
+<tr class="a">
+<td> <a href="../metron-solr/index.html#configuration"><tt>solr.commit.waitFlush</tt></a> </td>
+<td> Indexing </td>
+<td> String </td>
+<td> N/A </td></tr>
+<tr class="b">
+<td> <a href="../metron-solr/index.html#configuration"><tt>solr.collection</tt></a> </td>
+<td> Indexing </td>
+<td> String </td>
+<td> N/A </td></tr>
+<tr class="a">
+<td> <a href="../metron-solr/index.html#configuration"><tt>solr.http.config</tt></a> </td>
+<td> Indexing </td>
+<td> String </td>
+<td> N/A </td></tr>
+<tr class="b">
+<td> <a href="#validation-framework"><tt>fieldValidations</tt></a> </td>
<td> Parsing </td>
<td> Object </td>
-<td> N/A </td></tr>
-<tr class="b">
-<td> <a href="../metron-parsers/index.html#parser.error.topic"><tt>parser.error.topic</tt></a> </td>
+<td> N/A </td></tr>
+<tr class="a">
+<td> <a href="../metron-parsers/index.html#parser.error.topic"><tt>parser.error.topic</tt></a> </td>
<td> Parsing </td>
<td> String </td>
-<td> N/A </td></tr>
-<tr class="a">
-<td> <a href="../../metron-stellar/stellar-common/index.html#stellar.function.paths"><tt>stellar.function.paths</tt></a> </td>
-<td> Stellar </td>
-<td> CSV String </td>
-<td> N/A </td></tr>
+<td> <tt>parser_error_topic</tt> </td></tr>
<tr class="b">
-<td> <a href="../../metron-stellar/stellar-common/index.html#stellarfunctionresolverincludesexcludes"><tt>stellar.function.resolver.includes</tt></a> </td>
+<td> <a href="../../metron-stellar/stellar-common/index.html#stellar.function.paths"><tt>stellar.function.paths</tt></a> </td>
<td> Stellar </td>
<td> CSV String </td>
-<td> N/A </td></tr>
+<td> N/A </td></tr>
<tr class="a">
-<td> <a href="../../metron-stellar/stellar-common/index.html#stellarfunctionresolverincludesexcludes"><tt>stellar.function.resolver.excludes</tt></a> </td>
+<td> <a href="../../metron-stellar/stellar-common/index.html#stellarfunctionresolverincludesexcludes"><tt>stellar.function.resolver.includes</tt></a> </td>
<td> Stellar </td>
<td> CSV String </td>
-<td> N/A </td></tr>
+<td> N/A </td></tr>
<tr class="b">
-<td> <a href="../../metron-analytics/metron-profiler/index.html#profiler.period.duration"><tt>profiler.period.duration</tt></a> </td>
+<td> <a href="../../metron-stellar/stellar-common/index.html#stellarfunctionresolverincludesexcludes"><tt>stellar.function.resolver.excludes</tt></a> </td>
+<td> Stellar </td>
+<td> CSV String </td>
+<td> N/A </td></tr>
+<tr class="a">
+<td> <a href="../../metron-analytics/metron-profiler-storm/index.html#profiler.period.duration"><tt>profiler.period.duration</tt></a> </td>
<td> Profiler </td>
<td> Integer </td>
-<td> <tt>profiler_period_duration</tt> </td></tr>
-<tr class="a">
-<td> <a href="../../metron-analytics/metron-profiler/index.html#profiler.period.duration.units"><tt>profiler.period.duration.units</tt></a> </td>
+<td> <tt>profiler_period_duration</tt> </td></tr>
+<tr class="b">
+<td> <a href="../../metron-analytics/metron-profiler-storm/index.html#profiler.period.duration.units"><tt>profiler.period.duration.units</tt></a> </td>
<td> Profiler </td>
<td> String </td>
-<td> <tt>profiler_period_units</tt> </td></tr>
-<tr class="b">
-<td> <a href="../../metron-analytics/metron-profiler/index.html#profiler.writer.batchSize"><tt>profiler.writer.batchSize</tt></a> </td>
-<td> Profiler </td>
-<td> Integer </td>
-<td> N/A </td></tr>
+<td> <tt>profiler_period_units</tt> </td></tr>
<tr class="a">
-<td> <a href="../../metron-analytics/metron-profiler/index.html#profiler.writer.batchTimeout"><tt>profiler.writer.batchTimeout</tt></a> </td>
+<td> <a href="../../metron-analytics/metron-profiler-storm/index.html#profilerperiodduration"><tt>profiler.client.period.duration</tt></a> </td>
<td> Profiler </td>
<td> Integer </td>
-<td> N/A </td></tr>
+<td> <tt>profiler_period_duration</tt> </td></tr>
<tr class="b">
-<td> <a href="../metron-indexing/index.html#update.hbase.table"><tt>update.hbase.table</tt></a> </td>
+<td> <a href="../../metron-analytics/metron-profiler-storm/index.html#profilerperioddurationunits"><tt>profiler.client.period.duration.units</tt></a> </td>
+<td> Profiler </td>
+<td> String </td>
+<td> <tt>profiler_period_units</tt> </td></tr>
+<tr class="a">
+<td> <a href="../../metron-analytics/metron-profiler-storm/index.html#profiler.writer.batchSize"><tt>profiler.writer.batchSize</tt></a> </td>
+<td> Profiler </td>
+<td> Integer </td>
+<td> <tt>profiler_kafka_writer_batch_size</tt> </td></tr>
+<tr class="b">
+<td> <a href="../../metron-analytics/metron-profiler-storm/index.html#profiler.writer.batchTimeout"><tt>profiler.writer.batchTimeout</tt></a> </td>
+<td> Profiler </td>
+<td> Integer </td>
+<td> <tt>profiler_kafka_writer_batch_timeout</tt> </td></tr>
+<tr class="a">
+<td> <a href="../metron-indexing/index.html#update.hbase.table"><tt>update.hbase.table</tt></a> </td>
<td> REST/Indexing </td>
<td> String </td>
-<td> <tt>update_hbase_table</tt> </td></tr>
-<tr class="a">
-<td> <a href="../metron-indexing/index.html#update.hbase.cf"><tt>update.hbase.cf</tt></a> </td>
+<td> <tt>update_hbase_table</tt> </td></tr>
+<tr class="b">
+<td> <a href="../metron-indexing/index.html#update.hbase.cf"><tt>update.hbase.cf</tt></a> </td>
<td> REST/Indexing </td>
<td> String </td>
-<td> <tt>update_hbase_cf</tt> </td></tr>
+<td> <tt>update_hbase_cf</tt> </td></tr>
+<tr class="a">
+<td> <a href="../metron-interface/metron-rest/index.html"><tt>user.settings.hbase.table</tt></a> </td>
+<td> REST/Indexing </td>
+<td> String </td>
+<td> <tt>user_settings_hbase_table</tt> </td></tr>
<tr class="b">
-<td> <a href="../metron-enrichment/index.html#geo.hdfs.file"><tt>geo.hdfs.file</tt></a> </td>
+<td> <a href="../metron-interface/metron-rest/index.html"><tt>user.settings.hbase.cf</tt></a> </td>
+<td> REST/Indexing </td>
+<td> String </td>
+<td> <tt>user_settings_hbase_cf</tt> </td></tr>
+<tr class="a">
+<td> <a href="../metron-enrichment/metron-enrichment-common/index.html#geo.hdfs.file"><tt>geo.hdfs.file</tt></a> </td>
<td> Enrichment </td>
<td> String </td>
-<td> <tt>geo_hdfs_file</tt> </td></tr>
-<tr class="a">
-<td> <a href="../metron-enrichment/index.html#enrichment.writer.batchSize"><tt>enrichment.writer.batchSize</tt></a> </td>
+<td> <tt>geo_hdfs_file</tt> </td></tr>
+<tr class="b">
+<td> <a href="../metron-enrichment/metron-enrichment-common/index.html#enrichment.writer.batchSize"><tt>enrichment.writer.batchSize</tt></a> </td>
<td> Enrichment </td>
<td> Integer </td>
-<td> N/A </td></tr>
-<tr class="b">
-<td> <a href="../metron-enrichment/index.html#enrichment.writer.batchTimeout"><tt>enrichment.writer.batchTimeout</tt></a> </td>
+<td> <tt>enrichment_kafka_writer_batch_size</tt> </td></tr>
+<tr class="a">
+<td> <a href="../metron-enrichment/metron-enrichment-common/index.html#enrichment.writer.batchTimeout"><tt>enrichment.writer.batchTimeout</tt></a> </td>
<td> Enrichment </td>
<td> Integer </td>
-<td> N/A </td></tr>
-<tr class="a">
-<td> <a href="../metron-enrichment/index.html#geo.hdfs.file"><tt>geo.hdfs.file</tt></a> </td>
+<td> <tt>enrichment_kafka_writer_batch_timeout</tt> </td></tr>
+<tr class="b">
+<td> <a href="../metron-hbase-server/index.html#enrichment.list.hbase.provider.impl"><tt>enrichment.list.hbase.provider.impl</tt></a> </td>
<td> Enrichment </td>
<td> String </td>
-<td> <tt>geo_hdfs_file</tt> </td></tr>
-<tr class="b">
-<td> <a href="../../metron-interface/metron-alerts/index.html#source.type.field"><tt>source.type.field</tt></a> </td>
-<td> UI </td>
-<td> String </td>
-<td> <tt>source_type_field</tt> </td></tr>
+<td> <tt>enrichment_list_hbase_provider_impl</tt> </td></tr>
<tr class="a">
-<td> <a href="../../metron-interface/metron-alerts/index.html#threat.triage.score.field"><tt>threat.triage.score.field</tt></a> </td>
+<td> <a href="../metron-hbase-server/index.html#enrichment.list.hbase.table"><tt>enrichment.list.hbase.table</tt></a> </td>
+<td> Enrichment </td>
+<td> String </td>
+<td> <tt>enrichment_list_hbase_table</tt> </td></tr>
+<tr class="b">
+<td> <a href="../metron-hbase-server/index.html#enrichment.list.hbase.cf"><tt>enrichment.list.hbase.cf</tt></a> </td>
+<td> Enrichment </td>
+<td> String </td>
+<td> <tt>enrichment_list_hbase_cf</tt> </td></tr>
+<tr class="a">
+<td> <a href="../metron-enrichment/index.html#geo.hdfs.file"><tt>geo.hdfs.file</tt></a> </td>
+<td> Enrichment </td>
+<td> String </td>
+<td> <tt>geo_hdfs_file</tt> </td></tr>
+<tr class="b">
+<td> <a href="../../metron-interface/metron-alerts/index.html#source.type.field"><tt>source.type.field</tt></a> </td>
<td> UI </td>
<td> String </td>
-<td> <tt>threat_triage_score_field</tt> </td></tr>
+<td> <tt>source_type_field</tt> </td></tr>
+<tr class="a">
+<td> <a href="../../metron-interface/metron-alerts/index.html#threat.triage.score.field"><tt>threat.triage.score.field</tt></a> </td>
+<td> UI </td>
+<td> String </td>
+<td> <tt>threat_triage_score_field</tt> </td></tr>
</tbody>
</table>
<div class="section">
<h2><a name="Note_Configs_in_Ambari"></a>Note Configs in Ambari</h2>
<p>If a field is managed via ambari, you should change the field via ambari. Otherwise, upon service restarts, you may find your update overwritten.</p>
+<p><a name="High_Level_Architecture"></a></p>
+<h1>High Level Architecture</h1>
+<p>As already pointed out in the main project README, Apache Metron is a Kappa architecture (see <a href="../../index.html#Navigating_the_Architecture">Navigating the Architecture</a>) primarily backed by Storm and Kafka. We additionally leverage:</p>
+<ul>
+
+<li>Zookeeper for dynamic configuration updates to running Storm topologies. This enables us to push updates to our Storm topologies without restarting them.</li>
+<li>HBase primarily for enrichments. But we also use it to store user state for our UI’s.</li>
+<li>HDFS for long term storage. Our parsed and enriched messages land here, along with any reported exceptions or errors encountered along the way.</li>
+<li>Solr and Elasticsearch (plus Kibana) for real-time access. We provide out of the box compatibility with both Solr and Elasticsearch, and custom dashboards for data exploration in Kibana.</li>
+<li>Zeppelin for providing dashboards to do custom analytics.</li>
+</ul>
+<p>Getting data “into” Metron is accomplished by setting up a Kafka topic for parsers to read from. There are a variety of options, including, but not limited to:</p>
+<ul>
+
+<li><a class="externalLink" href="https://github.com/apache/metron-bro-plugin-kafka">Bro Kafka plugin</a></li>
+<li><a href="../../metron-sensors/fastcapa/index.html">Fastcapa</a></li>
+<li><a class="externalLink" href="https://nifi.apache.org">NiFi</a></li>
+</ul>
<p><a name="Validation_Framework"></a></p>
<h1>Validation Framework</h1>
<p>Inside of the global configuration, there is a validation framework in place that enables the validation that messages coming from all parsers are valid. This is done in the form of validation plugins where assertions about fields or whole messages can be made.</p>
@@ -530,7 +618,8 @@
"error_hash": "f7baf053f2d3c801a01d196f40f3468e87eea81788b2567423030100865c5061",
"error_type": "parser_error",
"message": "Unable to parse Message: {\"http\": {\"ts\":1488809627.000000.31915,\"uid\":\"C9JpSd2vFAWo3mXKz1\", ...",
- "timestamp": 1488809630698
+ "timestamp": 1488809630698,
+ "guid": "bf9fb8d1-2507-4a41-a5b2-42f75f6ddc63"
}
</pre></div></div>
@@ -639,9 +728,7 @@
Metron home directory
-p DIRECTORY, --hdp_home=DIRECTORY
HDP home directory
-</pre></div></div>
-
-<p>`</p></div></div>
+</pre></div></div></div></div>
</div>
</div>
</div>
diff --git a/current-book/metron-platform/metron-data-management/index.html b/current-book/metron-platform/metron-data-management/index.html
index 5f2195c..f6ab66e 100644
--- a/current-book/metron-platform/metron-data-management/index.html
+++ b/current-book/metron-platform/metron-data-management/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-data-management/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-data-management/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Resource Data Management</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Resource Data Management</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -59,10 +57,11 @@
<li class="active"><a href="#"><span class="none"></span>Data-management</a></li>
<li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
<li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
<li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
<li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
- <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
<li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
<li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
@@ -595,7 +594,7 @@
</table></div>
<div class="section">
<h3><a name="GeoLite2_Loader"></a>GeoLite2 Loader</h3>
-<p>The shell script <tt>$METRON_HOME/bin/geo_enrichment_load.sh</tt> will retrieve MaxMind GeoLite2 data and load data into HDFS, and update the configuration.</p>
+<p>The shell script <tt>$METRON_HOME/bin/maxmind_enrichment_load.sh</tt> will retrieve MaxMind GeoLite2 data and load data into HDFS, and update the configuration. The script will retrieve both the City and ASN databases.</p>
<p>THIS SCRIPT WILL NOT UPDATE AMBARI’S GLOBAL.JSON, JUST THE ZK CONFIGS. CHANGES WILL GO INTO EFFECT, BUT WILL NOT PERSIST PAST AN AMBARI RESTART UNTIL UPDATED THERE.</p>
<p>The parameters for the utility are as follows:</p>
<table border="0" class="table table-striped">
@@ -605,34 +604,44 @@
<th> Short Code </th>
<th> Long Code </th>
<th> Is Required? </th>
-<th> Description </th></tr>
+<th> Description </th></tr>
</thead><tbody>
<tr class="b">
<td> -h </td>
<td> </td>
<td> No </td>
-<td> Generate the help screen/set of options </td></tr>
+<td> Generate the help screen/set of options </td></tr>
<tr class="a">
<td> -g </td>
<td> –geo_url </td>
<td> No </td>
-<td> GeoIP URL - defaults to <a class="externalLink" href="http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz">http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz</a> </td></tr>
+<td> GeoIP URL - defaults to <a class="externalLink" href="http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz">http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz</a> </td></tr>
<tr class="b">
<td> -r </td>
<td> –remote_dir </td>
<td> No </td>
-<td> HDFS directory to land formatted GeoIP file - defaults to /apps/metron/geo/<epoch millis>/ </td></tr>
+<td> HDFS directory to land formatted GeoIP City file - defaults to /apps/metron/geo/<epoch millis>/ </td></tr>
+<tr class="a">
+<td> -ra </td>
+<td> –remote_asn_dir </td>
+<td> No </td>
+<td> HDFS directory to land formatted GeoIP ASN file - defaults to /apps/metron/asn/<epoch millis>/ </td></tr>
+<tr class="b">
+<td> -re </td>
+<td> –retries </td>
+<td> No </td>
+<td> Number of GeoLite2 database download retries, after an initial failure. </td></tr>
<tr class="a">
<td> -t </td>
<td> –tmp_dir </td>
<td> No </td>
-<td> Directory for landing the temporary GeoIP data - defaults to /tmp </td></tr>
+<td> Directory for landing the temporary GeoIP data - defaults to /tmp </td></tr>
<tr class="b">
<td> -z </td>
<td> –zk_quorum </td>
<td> Yes </td>
-<td> Zookeeper Quorum URL (zk1:port,zk2:port,…) </td></tr>
+<td> Zookeeper Quorum URL (zk1:port,zk2:port,…) </td></tr>
</tbody>
</table></div>
<div class="section">
diff --git a/current-book/metron-platform/metron-elasticsearch/index.html b/current-book/metron-platform/metron-elasticsearch/index.html
index 24ee901..7a6cc0a 100644
--- a/current-book/metron-platform/metron-elasticsearch/index.html
+++ b/current-book/metron-platform/metron-elasticsearch/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-elasticsearch/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-elasticsearch/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Elasticsearch in Metron</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Elasticsearch in Metron</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -59,10 +57,11 @@
<li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
<li class="active"><a href="#"><span class="none"></span>Elasticsearch</a></li>
<li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
<li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
<li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
- <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
<li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
<li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
@@ -112,9 +111,9 @@
<li><a href="#Introduction">Introduction</a></li>
<li><a href="#Properties">Properties</a></li>
-<li><a href="#Upgrading_to_5.6.2">Upgrading to 5.6.2</a></li>
+<li><a href="#Upgrading_from_2.3.3_to_5.6">Upgrading from 2.3.3 to 5.6</a></li>
<li><a href="#Type_Mappings">Type Mappings</a></li>
-<li><a href="#Using_Metron_with_Elasticsearch_5.6.2">Using Metron with Elasticsearch 5.6.2</a></li>
+<li><a href="#Using_Metron_with_Elasticsearch_5.6">Using Metron with Elasticsearch 5.6</a></li>
<li><a href="#Installing_Elasticsearch_Templates">Installing Elasticsearch Templates</a></li>
</ul></div>
<div class="section">
@@ -283,8 +282,8 @@
<li>Other notes on JSSE - <a class="externalLink" href="https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html">https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html</a></li>
</ul></div></div>
<div class="section">
-<h2><a name="Upgrading_to_5.6.2"></a>Upgrading to 5.6.2</h2>
-<p>Users should be prepared to re-index when migrating from Elasticsearch 2.3.3 to 5.6.2. There are a number of template changes, most notably around string type handling, that may cause issues when upgrading.</p>
+<h2><a name="Upgrading_from_2.3.3_to_5.6"></a>Upgrading from 2.3.3 to 5.6</h2>
+<p>Users should be prepared to re-index when migrating from Elasticsearch 2.3.3 to 5.6. There are a number of template changes, most notably around string type handling, that may cause issues when upgrading.</p>
<p><a class="externalLink" href="https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup-upgrade.html">https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setup-upgrade.html</a></p>
<p>Be aware that if you add a new string value and want to be able to filter and search on this value from the Alerts UI, you <b>must</b> add a mapping for that type to the appropriate Elasticsearch template. Below is more detail on how to choose the appropriate mapping type for your string value.</p></div>
<div class="section">
@@ -564,7 +563,7 @@
<li>metron_alert - nested</li>
</ul></div></div>
<div class="section">
-<h2><a name="Using_Metron_with_Elasticsearch_5.6.2"></a>Using Metron with Elasticsearch 5.6.2</h2>
+<h2><a name="Using_Metron_with_Elasticsearch_5.6"></a>Using Metron with Elasticsearch 5.6</h2>
<p>Although infrequent, sometimes an internal field is added in Metron and existing templates must be updated. The following steps outlines how to do this, using <tt>metron_alert</tt> as an example.</p>
<p>With the addition of the meta alert feature, there is a requirement that all sensors templates have a nested <tt>metron_alert</tt> field defined. This field is a dummy field. See <a class="externalLink" href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields">Ignoring Unmapped Fields</a> for more information</p>
<p>Without this field, an error will be thrown during ALL searches (including from UIs, resulting in no alerts being found for any sensor). This error will be found in the REST service’s logs.</p>
diff --git a/current-book/metron-platform/metron-enrichment/index.html b/current-book/metron-platform/metron-enrichment/index.html
index a6d9697..3f1700d 100644
--- a/current-book/metron-platform/metron-enrichment/index.html
+++ b/current-book/metron-platform/metron-enrichment/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-enrichment/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-enrichment/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Enrichment</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Enrichment</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -60,13 +58,15 @@
<li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
<li class="active"><a href="#"><span class="icon-chevron-down"></span>Enrichment</a>
<ul class="nav nav-list">
- <li><a href="../../metron-platform/metron-enrichment/Performance.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../metron-platform/metron-enrichment/metron-enrichment-common/index.html" title="Enrichment-common"><span class="none"></span>Enrichment-common</a></li>
+ <li><a href="../../metron-platform/metron-enrichment/metron-enrichment-storm/index.html" title="Enrichment-storm"><span class="icon-chevron-right"></span>Enrichment-storm</a></li>
</ul>
</li>
+ <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
<li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
<li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
- <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
<li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
<li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
@@ -112,7 +112,7 @@
<p><a name="Enrichment"></a></p>
<div class="section">
<h2><a name="Introduction"></a>Introduction</h2>
-<p>The <tt>enrichment</tt> topology is a topology dedicated to taking the data from the parsing topologies that have been normalized into the Metron data format (e.g. a JSON Map structure with <tt>original_message</tt> and <tt>timestamp</tt>) and</p>
+<p>The <tt>enrichment</tt> module is a module dedicated to taking the data from the parsers that have been normalized into the Metron data format (e.g. a JSON Map structure with <tt>original_message</tt> and <tt>timestamp</tt>) and</p>
<ul>
<li>Enriching messages with external data from data stores (e.g. hbase) by adding new fields based on existing fields in the messages.</li>
@@ -120,432 +120,20 @@
<li>Marking threat alerts with a numeric triage level based on a set of Stellar rules.</li>
</ul></div>
<div class="section">
-<h2><a name="Enrichment_Architecture"></a>Enrichment Architecture</h2>
-<p><img src="../../images/unified_enrichment_arch.svg" alt="Unified Architecture" /></p>
+<h2><a name="Deployment_Options"></a>Deployment Options</h2>
+<p>There is currently one option for running enrichments in Metron, which is as a Storm topology.</p></div>
<div class="section">
-<h3><a name="Unified_Enrichment_Topology"></a>Unified Enrichment Topology</h3>
-<p>The unified enrichment topology uses data parallelism as opposed to the deprecated split/join topology’s task parallelism. This architecture uses a worker pool to fully enrich any message within a worker. This results in</p>
+<h2><a name="Submodules"></a>Submodules</h2>
<ul>
-<li>Fewer bolts in the topology</li>
-<li>Each bolt fully operates on a message.</li>
-<li>Fewer network hops</li>
-</ul>
-<p>This architecture is fully backwards compatible with the old split-join topology; the only difference is how the enrichment will operate on each message (in one bolt where the split/join is done in a threadpool as opposed to split across multiple bolts).</p>
-<div class="section">
-<h4><a name="Configuring_It"></a>Configuring It</h4>
-<p>There are two parameters which you might want to tune in this topology. Both of them are topology configuration adjustable in the flux file <tt>$METRON_HOME/config/flux/enrichment/remote-unified.yaml</tt>:</p>
-<ul>
-
-<li><tt>metron.threadpool.size</tt> : The size of the threadpool. This can take a number or a multiple of the number of cores (e.g. <tt>5C</tt> to 5 times the number of cores). The default is <tt>2C</tt>.</li>
-<li><tt>metron.threadpool.type</tt> : The type of threadpool. (note: descriptions taken from <a class="externalLink" href="https://zeroturnaround.com/rebellabs/fixedthreadpool-cachedthreadpool-or-forkjoinpool-picking-correct-java-executors-for-background-tasks/">here</a>).
-<ul>
-
-<li><tt>FIXED</tt> is a fixed threadpool of size <tt>n</tt>. <tt>n</tt> threads will process tasks at the time, when the pool is saturated, new tasks will get added to a queue without a limit on size. Good for CPU intensive tasks. This is the default.</li>
-<li><tt>WORK_STEALING</tt> is a work stealing threadpool. This will create and shut down threads dynamically to accommodate the required parallelism level. It also tries to reduce the contention on the task queue, so can be really good in heavily loaded environments. Also good when your tasks create more tasks for the executor, like recursive tasks.</li>
-</ul>
-</li>
-</ul>
-<p>In order to configure the parallelism for the enrichment bolt and threat intel bolt, the configurations will be taken from the respective join bolt parallelism. When proper ambari support for this is added, we will add its own property.</p></div></div>
-<div class="section">
-<h3><a name="Split-Join_Enrichment_Topology"></a>Split-Join Enrichment Topology</h3>
-<p>The now-deprecated split/join topology is also available and performs enrichments in parallel. This poses some issues in terms of ease of tuning and reasoning about performance.</p>
-<p><img src="../../images/enrichment_arch.png" alt="Architecture" /></p>
-<div class="section">
-<h4><a name="Using_It"></a>Using It</h4>
-<p>In order to use the older, deprecated topology, you will need to</p>
-<ul>
-
-<li>Edit <tt>$METRON_HOME/bin/start_enrichment_topology.sh</tt> and adjust it to use <tt>remote-splitjoin.yaml</tt> instead of <tt>remote-unified.yaml</tt></li>
-<li>Restart the enrichment topology.</li>
-</ul></div></div></div>
-<div class="section">
-<h2><a name="Enrichment_Configuration"></a>Enrichment Configuration</h2>
-<p>The configuration for the <tt>enrichment</tt> topology, the topology primarily responsible for enrichment and threat intelligence enrichment, is defined by JSON documents stored in zookeeper.</p>
-<p>There are two types of configurations at the moment, <tt>global</tt> and <tt>sensor</tt> specific.</p></div>
-<div class="section">
-<h2><a name="Global_Configuration"></a>Global Configuration</h2>
-<p>There are a few enrichments which have independent configurations, such as from the global config. You can also configure the enrichment topology’s writer batching settings.</p>
-<p>Also, see the “<a href="../metron-common/index.html">Global Configuration</a>” section for more discussion of the global config.</p>
-<div class="section">
-<h3><a name="GeoIP"></a>GeoIP</h3>
-<p>Metron supports enrichment of IP information using <a class="externalLink" href="https://dev.maxmind.com/geoip/geoip2/geolite2/">GeoLite2</a>. The location of the file is managed in the global config.</p>
-<div class="section">
-<h4><a name="geo.hdfs.file"></a><tt>geo.hdfs.file</tt></h4>
-<p>The location on HDFS of the GeoLite2 database file to use for GeoIP lookups. This file will be localized on the storm supervisors running the topology and used from there. This is lazy, so if this property changes in a running topology, the file will be localized from HDFS upon first time the file is used via the geo enrichment.</p></div></div>
-<div class="section">
-<h3><a name="Writer_Batching"></a>Writer Batching</h3>
-<div class="section">
-<h4><a name="enrichment.writer.batchSize"></a><tt>enrichment.writer.batchSize</tt></h4>
-<p>The size of the batch that is written to Kafka at once. Defaults to <tt>15</tt> (size of 1 disables batching).</p></div>
-<div class="section">
-<h4><a name="enrichment.writer.batchTimeout"></a><tt>enrichment.writer.batchTimeout</tt></h4>
-<p>The timeout after which a batch will be flushed even if batchSize has not been met. Optional. If unspecified, or set to <tt>0</tt>, it defaults to a system-determined duration which is a fraction of the Storm parameter <tt>topology.message.timeout.secs</tt>. Ignored if batchSize is <tt>1</tt>, since this disables batching.</p></div></div></div>
-<div class="section">
-<h2><a name="Sensor_Enrichment_Configuration"></a>Sensor Enrichment Configuration</h2>
-<p>The sensor specific configuration is intended to configure the individual enrichments and threat intelligence enrichments for a given sensor type (e.g. <tt>snort</tt>).</p>
-<p>Just like the global config, the format is a JSON stored in zookeeper. The configuration is a complex JSON object with the following top level fields:</p>
-<ul>
-
-<li><tt>enrichment</tt> : A complex JSON object representing the configuration of the enrichments</li>
-<li><tt>threatIntel</tt> : A complex JSON object representing the configuration of the threat intelligence enrichments</li>
-</ul>
-<div class="section">
-<h3><a name="The_enrichment_Configuration"></a>The <tt>enrichment</tt> Configuration</h3>
-<table border="0" class="table table-striped">
-<thead>
-
-<tr class="a">
-<th> Field </th>
-<th> Description </th>
-<th> Example </th></tr>
-</thead><tbody>
-
-<tr class="b">
-<td> <tt>fieldToTypeMap</tt> </td>
-<td> In the case of a simple HBase enrichment (i.e. a key/value lookup), the mapping between fields and the enrichment types associated with those fields must be known. This enrichment type is used as part of the HBase key. Note: applies to hbaseEnrichment only. </td>
-<td> <tt>"fieldToTypeMap" : { "ip_src_addr" : [ "asset_enrichment" ] }</tt> </td></tr>
-<tr class="a">
-<td> <tt>fieldMap</tt> </td>
-<td> The map of enrichment bolts names to configuration handlers which know how to split the message up. The simplest of which is just a list of fields. More complex examples would be the stellar enrichment which provides stellar statements. Each field listed in the array arg is sent to the enrichment referenced in the key. Cardinality of fields to enrichments is many-to-many. </td>
-<td> <tt>"fieldMap": {"hbaseEnrichment": ["ip_src_addr","ip_dst_addr"]}</tt> </td></tr>
-<tr class="b">
-<td> <tt>config</tt> </td>
-<td> The general configuration for the enrichment </td>
-<td> <tt>"config": {"typeToColumnFamily": { "asset_enrichment" : "cf" } }</tt> </td></tr>
-</tbody>
-</table>
-<p>The <tt>config</tt> map is intended to house enrichment specific configuration. For instance, for the <tt>hbaseEnrichment</tt>, the mappings between the enrichment types to the column families is specified.</p>
-<p>The <tt>fieldMap</tt>contents are of interest because they contain the routing and configuration information for the enrichments.<br />
-When we say ‘routing’, we mean how the messages get split up and sent to the enrichment adapter bolts.<br />
-The simplest, by far, is just providing a simple list as in</p>
-
-<div>
-<div>
-<pre class="source"> "fieldMap": {
- "geo": [
- "ip_src_addr",
- "ip_dst_addr"
- ],
- "host": [
- "ip_src_addr",
- "ip_dst_addr"
- ],
- "hbaseEnrichment": [
- "ip_src_addr",
- "ip_dst_addr"
- ]
- }
-</pre></div></div>
-
-<p>Based on this sample config, both <tt>ip_src_addr</tt> and <tt>ip_dst_addr</tt> will go to the <tt>geo</tt>, <tt>host</tt>, and <tt>hbaseEnrichment</tt> adapter bolts.</p>
-<div class="section">
-<h4><a name="Stellar_Enrichment_Configuration"></a>Stellar Enrichment Configuration</h4>
-<p>For the <tt>geo</tt>, <tt>host</tt> and <tt>hbaseEnrichment</tt>, this is sufficient. However, more complex enrichments may contain their own configuration. Currently, the <tt>stellar</tt> enrichment is more adaptable and thus requires a more nuanced configuration.</p>
-<p>At its most basic, we want to take a message and apply a couple of enrichments, such as converting the <tt>hostname</tt> field to lowercase. We do this by specifying the transformation inside of the <tt>config</tt> for the <tt>stellar</tt> fieldMap. There are two syntaxes that are supported, specifying the transformations as a map with the key as the field and the value the stellar expression:</p>
-
-<div>
-<div>
-<pre class="source"> "fieldMap": {
- ...
- "stellar" : {
- "config" : {
- "hostname" : "TO_LOWER(hostname)"
- }
- }
- }
-</pre></div></div>
-
-<p>Another approach is to make the transformations as a list with the same <tt>var := expr</tt> syntax as is used in the Stellar REPL, such as:</p>
-
-<div>
-<div>
-<pre class="source"> "fieldMap": {
- ...
- "stellar" : {
- "config" : [
- "hostname := TO_LOWER(hostname)"
- ]
- }
- }
-</pre></div></div>
-
-<p>Sometimes arbitrary stellar enrichments may take enough time that you would prefer to split some of them into groups and execute the groups of stellar enrichments in parallel. Take, for instance, if you wanted to do an HBase enrichment and a profiler call which were independent of one another. This usecase is supported by splitting the enrichments up as groups.</p>
-<p>Consider the following example:</p>
-
-<div>
-<div>
-<pre class="source"> "fieldMap": {
- ...
- "stellar" : {
- "config" : {
- "malicious_domain_enrichment" : {
- "is_bad_domain" : "ENRICHMENT_EXISTS('malicious_domains', ip_dst_addr, 'enrichments', 'cf')"
- },
- "login_profile" : [
- "profile_window := PROFILE_WINDOW('from 6 months ago')",
- "global_login_profile := PROFILE_GET('distinct_login_attempts', 'global', profile_window)",
- "stats := STATS_MERGE(global_login_profile)",
- "auth_attempts_median := STATS_PERCENTILE(stats, 0.5)",
- "auth_attempts_sd := STATS_SD(stats)",
- "profile_window := null",
- "global_login_profile := null",
- "stats := null"
- ]
- }
- }
- }
-</pre></div></div>
-
-<p>Here we want to perform two enrichments that hit HBase and we would rather not run in sequence. These enrichments are entirely independent of one another (i.e. neither relies on the output of the other). In this case, we’ve created a group called <tt>malicious_domain_enrichment</tt> to inquire about whether the destination address exists in the HBase enrichment table in the <tt>malicious_domains</tt> enrichment type. This is a simple enrichment, so we can express the enrichment group as a map with the new field <tt>is_bad_domain</tt> being a key and the stellar expression associated with that operation being the associated value.</p>
-<p>In contrast, the stellar enrichment group <tt>login_profile</tt> is interacting with the profiler, has multiple temporary expressions (i.e. <tt>profile_window</tt>, <tt>global_login_profile</tt>, and <tt>stats</tt>) that are useful only within the context of this group of stellar expressions. In this case, we would need to ensure that we use the list construct when specifying the group and remember to set the temporary variables to <tt>null</tt> so they are not passed along.</p>
-<p>In general, things to note from this section are as follows:</p>
-<ul>
-
-<li>The stellar enrichments for the <tt>stellar</tt> enrichment adapter are specified in the <tt>config</tt> for the <tt>stellar</tt> enrichment adapter in the <tt>fieldMap</tt></li>
-<li>Groups of independent (i.e. no expression in any group depend on the output of an expression from an other group) may be executed in parallel</li>
-<li>If you have the need to use temporary variables, you may use the list construct. Ensure that you assign the variables to <tt>null</tt> before the end of the group.</li>
-<li><b>Ensure that you do not assign a field to a stellar expression which returns an object which JSON cannot represent.</b></li>
-<li>Fields assigned to Maps as part of stellar enrichments have the maps unfolded, similar to the HBase Enrichment
-<ul>
-
-<li>For example the stellar enrichment for field <tt>foo</tt> which assigns a map such as <tt>foo := { 'grok' : 1, 'bar' : 'baz'}</tt> would yield the following fields:
-<ul>
-
-<li><tt>foo.grok</tt> == <tt>1</tt></li>
-<li><tt>foo.bar</tt> == <tt>'baz'</tt></li>
-</ul>
-</li>
-</ul>
-</li>
-</ul></div></div>
-<div class="section">
-<h3><a name="The_threatIntel_Configuration"></a>The <tt>threatIntel</tt> Configuration</h3>
-<table border="0" class="table table-striped">
-<thead>
-
-<tr class="a">
-<th> Field </th>
-<th> Description </th>
-<th> Example </th></tr>
-</thead><tbody>
-
-<tr class="b">
-<td> <tt>fieldToTypeMap</tt> </td>
-<td> In the case of a simple HBase threat intel enrichment (i.e. a key/value lookup), the mapping between fields and the enrichment types associated with those fields must be known. This enrichment type is used as part of the HBase key. Note: applies to hbaseThreatIntel only. </td>
-<td> <tt>"fieldToTypeMap" : { "ip_src_addr" : [ "malicious_ips" ] }</tt> </td></tr>
-<tr class="a">
-<td> <tt>fieldMap</tt> </td>
-<td> The map of threat intel enrichment bolts names to fields in the JSON messages. Each field is sent to the threat intel enrichment bolt referenced in the key. Each field listed in the array arg is sent to the enrichment referenced in the key. Cardinality of fields to enrichments is many-to-many. </td>
-<td> <tt>"fieldMap": {"hbaseThreatIntel": ["ip_src_addr","ip_dst_addr"]}</tt> </td></tr>
-<tr class="b">
-<td> <tt>triageConfig</tt> </td>
-<td> The configuration of the threat triage scorer. In the situation where a threat is detected, a score is assigned to the message and embedded in the indexed message. </td>
-<td> <tt>"riskLevelRules" : { "IN_SUBNET(ip_dst_addr, '192.168.0.0/24')" : 10 }</tt> </td></tr>
-<tr class="a">
-<td> <tt>config</tt> </td>
-<td> The general configuration for the Threat Intel </td>
-<td> <tt>"config": {"typeToColumnFamily": { "malicious_ips","cf" } }</tt> </td></tr>
-</tbody>
-</table>
-<p>The <tt>config</tt> map is intended to house threat intel specific configuration. For instance, for the <tt>hbaseThreatIntel</tt> threat intel adapter, the mappings between the enrichment types to the column families is specified. The <tt>fieldMap</tt> configuration is similar to the <tt>enrichment</tt> configuration in that the adapters available are the same.</p>
-<p>The <tt>triageConfig</tt> field is also a complex field and it bears some description:</p>
-<table border="0" class="table table-striped">
-<thead>
-
-<tr class="a">
-<th> Field </th>
-<th> Description </th>
-<th> Example </th></tr>
-</thead><tbody>
-
-<tr class="b">
-<td> <tt>riskLevelRules</tt> </td>
-<td> This is a list of rules (represented as Stellar expressions) associated with scores with optional names and comments </td>
-<td> see below</td></tr>
-<tr class="a">
-<td> <tt>aggregator</tt> </td>
-<td> An aggregation function that takes all non-zero scores representing the matching queries from <tt>riskLevelRules</tt> and aggregates them into a single score. </td>
-<td> <tt>"MAX"</tt> </td></tr>
-</tbody>
-</table>
-<p>A risk level rule is of the following format:</p>
-<ul>
-
-<li><tt>name</tt> : The name of the threat triage rule</li>
-<li><tt>comment</tt> : A comment describing the rule</li>
-<li><tt>rule</tt> : The rule, represented as a Stellar statement</li>
-<li><tt>score</tt> : Associated threat triage score for the rule</li>
-<li><tt>reason</tt> : Reason the rule tripped. Can be represented as a Stellar statement</li>
-</ul>
-<p>An example of a rule is as follows:</p>
-
-<div>
-<div>
-<pre class="source"> "riskLevelRules" : [
- {
- "name" : "is internal"
- , "comment" : "determines if the destination is internal."
- , "rule" : "IN_SUBNET(ip_dst_addr, '192.168.0.0/24')"
- , "score" : 10
- , "reason" : "FORMAT('%s is internal', ip_dst_addr)"
- }
- ]
-</pre></div></div>
-
-<p>The supported aggregation functions are:</p>
-<ul>
-
-<li><tt>MAX</tt> : The max of all of the associated values for matching queries</li>
-<li><tt>MIN</tt> : The min of all of the associated values for matching queries</li>
-<li><tt>MEAN</tt> : The mean of all of the associated values for matching queries</li>
-<li><tt>SUM</tt> : The sum of all the associated values for matching queries</li>
-<li><tt>POSITIVE_MEAN</tt> : The mean of the positive associated values for the matching queries.</li>
+<li>metron-enrichment-common - this module houses the prepackaged enrichment configuration by sensor. It also contains the core enrichment and threat intelligence processing functionality.</li>
+<li>metron-common-storm - this module is home to Storm-specific code such as Flux files and Storm Bolts.</li>
</ul></div>
<div class="section">
-<h3><a name="Example_Configuration"></a>Example Configuration</h3>
-<p>An example configuration for the YAF sensor is as follows:</p>
-
-<div>
-<div>
-<pre class="source">{
- "enrichment": {
- "fieldMap": {
- "geo": [
- "ip_src_addr",
- "ip_dst_addr"
- ],
- "host": [
- "ip_src_addr",
- "ip_dst_addr"
- ],
- "hbaseEnrichment": [
- "ip_src_addr",
- "ip_dst_addr"
- ]
- }
- ,"fieldToTypeMap": {
- "ip_src_addr": [
- "playful_classification"
- ],
- "ip_dst_addr": [
- "playful_classification"
- ]
- }
- },
- "threatIntel": {
- "fieldMap": {
- "hbaseThreatIntel": [
- "ip_src_addr",
- "ip_dst_addr"
- ]
- },
- "fieldToTypeMap": {
- "ip_src_addr": [
- "malicious_ip"
- ],
- "ip_dst_addr": [
- "malicious_ip"
- ]
- },
- "triageConfig" : {
- "riskLevelRules" : [
- {
- "rule" : "ip_src_addr == '10.0.2.3' or ip_dst_addr == '10.0.2.3'",
- "score" : 10
- }
- ],
- "aggregator" : "MAX"
- }
- }
-}
-</pre></div></div>
-
-<p>ThreatIntel alert levels are emitted as a new field “threat.triage.level.” So for the example above, an incoming message that trips the <tt>ip_src_addr</tt> rule will have a new field threat.triage.level=10.</p>
-<p><a name="Example_Enrichment_via_Stellar"></a></p>
-<h1>Example Enrichment via Stellar</h1>
-<p>Let’s walk through doing a simple enrichment using Stellar on your cluster using the Squid topology.</p></div></div>
-<div class="section">
-<h2><a name="Install_Prerequisites"></a>Install Prerequisites</h2>
-<p>Now let’s install some prerequisites:</p>
-<ul>
-
-<li>Squid client via <tt>yum install squid</tt></li>
-<li>ES Head plugin via <tt>/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head</tt></li>
-</ul>
-<p>Start Squid via <tt>service squid start</tt></p></div>
-<div class="section">
-<h2><a name="Adjust_Enrichment_Configurations_for_Squid_to_Call_Stellar"></a>Adjust Enrichment Configurations for Squid to Call Stellar</h2>
-<p>Let’s adjust the configurations for the Squid topology to annotate the messages using some Stellar functions.</p>
-<ul>
-
-<li>Edit the squid enrichment configuration at <tt>$METRON_HOME/config/zookeeper/enrichments/squid.json</tt> (this file will not exist, so create a new one) to add some new fields based on stellar queries:</li>
-</ul>
-
-<div>
-<div>
-<pre class="source">{
- "enrichment" : {
- "fieldMap": {
- "stellar" : {
- "config" : {
- "numeric" : {
- "foo": "1 + 1"
- }
- ,"ALL_CAPS" : "TO_UPPER(source.type)"
- }
- }
- }
- },
- "threatIntel" : {
- "fieldMap":{
- "stellar" : {
- "config" : {
- "bar" : "TO_UPPER(source.type)"
- }
- }
- },
- "triageConfig" : {
- }
- }
-}
-</pre></div></div>
-
-<p>We have added the following fields as part of the enrichment phase of the enrichment topology:</p>
-<ul>
-
-<li><tt>foo</tt> == 2</li>
-<li><tt>ALL_CAPS</tt> == SQUID</li>
-</ul>
-<p>We have added the following as part of the threat intel:</p>
-<ul>
-
-<li><tt>bar</tt> == SQUID</li>
-</ul>
-<p>Please note that foo and ALL_CAPS will be applied in separate workers due to them being in separate groups.</p>
-<ul>
-
-<li>Upload new configs via <tt>$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -i $METRON_HOME/config/zookeeper -z node1:2181</tt></li>
-<li>Make the Squid topic in kafka via <tt>/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper node1:2181 --create --topic squid --partitions 1 --replication-factor 1</tt></li>
-</ul></div>
-<div class="section">
-<h2><a name="Start_Topologies_and_Send_Data"></a>Start Topologies and Send Data</h2>
-<p>Now we need to start the topologies and send some data:</p>
-<ul>
-
-<li>Start the squid topology via <tt>$METRON_HOME/bin/start_parser_topology.sh -k node1:6667 -z node1:2181 -s squid</tt></li>
-<li>Generate some data via the squid client:
-<ul>
-
-<li><tt>squidclient http://yahoo.com</tt></li>
-<li><tt>squidclient http://cnn.com</tt></li>
-</ul>
-</li>
-<li>Send the data to kafka via <tt>cat /var/log/squid/access.log | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic squid</tt></li>
-<li>Browse the data in elasticsearch via the ES Head plugin @ <a class="externalLink" href="http://node1:9200/_plugin/head/">http://node1:9200/_plugin/head/</a> and verify that in the squid index you have two documents</li>
-<li>Ensure that the documents have new fields <tt>foo</tt>, <tt>bar</tt> and <tt>ALL_CAPS</tt> with values as described above.</li>
-</ul>
-<p>Note that we could have used any Stellar statements here, including calling out to HBase via <tt>ENRICHMENT_GET</tt> and <tt>ENRICHMENT_EXISTS</tt> or even calling a machine learning model via <a href="../../metron-analytics/metron-maas-service/index.html">Model as a Service</a>.</p></div>
+<h2><a name="Enrichments_List"></a>Enrichments List</h2>
+<p>Metron provides an HBase table for storing enrichments. The rowkeys are a combination of a salt (for managing RegionServer hotspotting), indicator (this would be the search value, e.g. “192.168.1.1”), and type (whois, geoip, etc.).</p>
+<p>This approach performs well for both inserts and lookups, but poses a challenge when looking to get an up-to-date list of all the current enrichments. This is of particular concern for the Management UI where it’s desirable to provide an accurate list of all available enrichment types. A table scan is undesirable because it results in a performance hit for inserts and reads. The alternative approach that mitigates these performance bottlenecks is to leverage a custom HBase Coprocessor which will listen to postPut calls from the RegionServer, extract the enrichment type from the rowkey, and perform an insert into a separate <tt>enrichment_list</tt> HBase table.</p>
+<p>See more about configuring the coprocessor here <a href="../metron-hbase-server/index.html#Enrichment_Coprocessor">Enrichment Coprocessor</a></p></div>
</div>
</div>
</div>
diff --git a/current-book/metron-platform/metron-enrichment/metron-enrichment-common/index.html b/current-book/metron-platform/metron-enrichment/metron-enrichment-common/index.html
new file mode 100644
index 0000000..f199370
--- /dev/null
+++ b/current-book/metron-platform/metron-enrichment/metron-enrichment-common/index.html
@@ -0,0 +1,431 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-enrichment/metron-enrichment-common/index.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Enrichment Common</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">Enrichment Common</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-down"></span>Enrichment</a>
+ <ul class="nav nav-list">
+ <li class="active"><a href="#"><span class="none"></span>Enrichment-common</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/metron-enrichment-storm/index.html" title="Enrichment-storm"><span class="icon-chevron-right"></span>Enrichment-storm</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
+ <li><a href="../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Enrichment Common</h1>
+<p><a name="Enrichment_Common"></a></p>
+<div class="section">
+<h2><a name="Introduction"></a>Introduction</h2>
+<p>This module holds shared code for enrichment activities.</p></div>
+<div class="section">
+<h2><a name="Enrichment_Configuration"></a>Enrichment Configuration</h2>
+<p>The configuration for the <tt>enrichment</tt> module, the module primarily responsible for enrichment and threat intelligence enrichment, is defined by JSON documents stored in zookeeper.</p>
+<p>There are two types of configurations at the moment, <tt>global</tt> and <tt>sensor</tt> specific.</p></div>
+<div class="section">
+<h2><a name="Global_Configuration"></a>Global Configuration</h2>
+<p>There are a few enrichments which have independent configurations, such as from the global config. You can also configure the enrichment topology’s writer batching settings.</p>
+<p>Also, see the “<a href="../../metron-common/index.html">Global Configuration</a>” section for more discussion of the global config.</p>
+<div class="section">
+<h3><a name="GeoIP"></a>GeoIP</h3>
+<p>Metron supports enrichment of IP information using <a class="externalLink" href="https://dev.maxmind.com/geoip/geoip2/geolite2/">GeoLite2</a>. The location of the file is managed in the global config.</p>
+<div class="section">
+<h4><a name="geo.hdfs.file"></a><tt>geo.hdfs.file</tt></h4>
+<p>The location on HDFS of the GeoLite2 database file to use for GeoIP lookups. This file will be localized on the storm supervisors running the topology and used from there. This is lazy, so if this property changes in a running topology, the file will be localized from HDFS upon first time the file is used via the geo enrichment.</p></div></div>
+<div class="section">
+<h3><a name="Writer_Batching"></a>Writer Batching</h3>
+<div class="section">
+<h4><a name="enrichment.writer.batchSize"></a><tt>enrichment.writer.batchSize</tt></h4>
+<p>The size of the batch that is written to Kafka at once. Defaults to <tt>15</tt> (size of 1 disables batching).</p></div>
+<div class="section">
+<h4><a name="enrichment.writer.batchTimeout"></a><tt>enrichment.writer.batchTimeout</tt></h4>
+<p>The timeout after which a batch will be flushed even if batchSize has not been met. Optional. If unspecified, or set to <tt>0</tt>, it defaults to a system-determined duration which is a fraction of the Storm parameter <tt>topology.message.timeout.secs</tt>. Ignored if batchSize is <tt>1</tt>, since this disables batching.</p></div></div></div>
+<div class="section">
+<h2><a name="Sensor_Enrichment_Configuration"></a>Sensor Enrichment Configuration</h2>
+<p>The sensor specific configuration is intended to configure the individual enrichments and threat intelligence enrichments for a given sensor type (e.g. <tt>snort</tt>).</p>
+<p>Just like the global config, the format is a JSON stored in zookeeper. The configuration is a complex JSON object with the following top level fields:</p>
+<ul>
+
+<li><tt>enrichment</tt> : A complex JSON object representing the configuration of the enrichments</li>
+<li><tt>threatIntel</tt> : A complex JSON object representing the configuration of the threat intelligence enrichments</li>
+</ul>
+<div class="section">
+<h3><a name="The_enrichment_Configuration"></a>The <tt>enrichment</tt> Configuration</h3>
+<table border="0" class="table table-striped">
+<thead>
+
+<tr class="a">
+<th> Field </th>
+<th> Description </th>
+<th> Example </th></tr>
+</thead><tbody>
+
+<tr class="b">
+<td> <tt>fieldToTypeMap</tt> </td>
+<td> In the case of a simple HBase enrichment (i.e. a key/value lookup), the mapping between fields and the enrichment types associated with those fields must be known. This enrichment type is used as part of the HBase key. Note: applies to hbaseEnrichment only. </td>
+<td> <tt>"fieldToTypeMap" : { "ip_src_addr" : [ "asset_enrichment" ] }</tt> </td></tr>
+<tr class="a">
+<td> <tt>fieldMap</tt> </td>
+<td> The map of enrichment bolts names to configuration handlers which know how to split the message up. The simplest of which is just a list of fields. More complex examples would be the stellar enrichment which provides stellar statements. Each field listed in the array arg is sent to the enrichment referenced in the key. Cardinality of fields to enrichments is many-to-many. </td>
+<td> <tt>"fieldMap": {"hbaseEnrichment": ["ip_src_addr","ip_dst_addr"]}</tt> </td></tr>
+<tr class="b">
+<td> <tt>config</tt> </td>
+<td> The general configuration for the enrichment </td>
+<td> <tt>"config": {"typeToColumnFamily": { "asset_enrichment" : "cf" } }</tt> </td></tr>
+</tbody>
+</table>
+<p>The <tt>config</tt> map is intended to house enrichment specific configuration. For instance, for the <tt>hbaseEnrichment</tt>, the mappings between the enrichment types to the column families is specified.</p>
+<p>The <tt>fieldMap</tt>contents are of interest because they contain the routing and configuration information for the enrichments.<br />
+When we say ‘routing’, we mean how the messages get split up and sent to the enrichment adapters. The simplest, by far, is just providing a simple list as in</p>
+
+<div>
+<div>
+<pre class="source"> "fieldMap": {
+ "geo": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ],
+ "host": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ],
+ "hbaseEnrichment": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ]
+ }
+</pre></div></div>
+
+<p>Based on this sample config, both <tt>ip_src_addr</tt> and <tt>ip_dst_addr</tt> will go to the <tt>geo</tt>, <tt>host</tt>, and <tt>hbaseEnrichment</tt> adapters.</p>
+<div class="section">
+<h4><a name="Stellar_Enrichment_Configuration"></a>Stellar Enrichment Configuration</h4>
+<p>For the <tt>geo</tt>, <tt>host</tt> and <tt>hbaseEnrichment</tt>, this is sufficient. However, more complex enrichments may contain their own configuration. Currently, the <tt>stellar</tt> enrichment is more adaptable and thus requires a more nuanced configuration.</p>
+<p>At its most basic, we want to take a message and apply a couple of enrichments, such as converting the <tt>hostname</tt> field to lowercase. We do this by specifying the transformation inside of the <tt>config</tt> for the <tt>stellar</tt> fieldMap. There are two syntaxes that are supported, specifying the transformations as a map with the key as the field and the value the stellar expression:</p>
+
+<div>
+<div>
+<pre class="source"> "fieldMap": {
+ ...
+ "stellar" : {
+ "config" : {
+ "hostname" : "TO_LOWER(hostname)"
+ }
+ }
+ }
+</pre></div></div>
+
+<p>Another approach is to make the transformations as a list with the same <tt>var := expr</tt> syntax as is used in the Stellar REPL, such as:</p>
+
+<div>
+<div>
+<pre class="source"> "fieldMap": {
+ ...
+ "stellar" : {
+ "config" : [
+ "hostname := TO_LOWER(hostname)"
+ ]
+ }
+ }
+</pre></div></div>
+
+<p>Sometimes arbitrary stellar enrichments may take enough time that you would prefer to split some of them into groups and execute the groups of stellar enrichments in parallel. Take, for instance, if you wanted to do an HBase enrichment and a profiler call which were independent of one another. This usecase is supported by splitting the enrichments up as groups.</p>
+<p>Consider the following example:</p>
+
+<div>
+<div>
+<pre class="source"> "fieldMap": {
+ ...
+ "stellar" : {
+ "config" : {
+ "malicious_domain_enrichment" : {
+ "is_bad_domain" : "ENRICHMENT_EXISTS('malicious_domains', ip_dst_addr, 'enrichments', 'cf')"
+ },
+ "login_profile" : [
+ "profile_window := PROFILE_WINDOW('from 6 months ago')",
+ "global_login_profile := PROFILE_GET('distinct_login_attempts', 'global', profile_window)",
+ "stats := STATS_MERGE(global_login_profile)",
+ "auth_attempts_median := STATS_PERCENTILE(stats, 0.5)",
+ "auth_attempts_sd := STATS_SD(stats)",
+ "profile_window := null",
+ "global_login_profile := null",
+ "stats := null"
+ ]
+ }
+ }
+ }
+</pre></div></div>
+
+<p>Here we want to perform two enrichments that hit HBase and we would rather not run in sequence. These enrichments are entirely independent of one another (i.e. neither relies on the output of the other). In this case, we’ve created a group called <tt>malicious_domain_enrichment</tt> to inquire about whether the destination address exists in the HBase enrichment table in the <tt>malicious_domains</tt> enrichment type. This is a simple enrichment, so we can express the enrichment group as a map with the new field <tt>is_bad_domain</tt> being a key and the stellar expression associated with that operation being the associated value.</p>
+<p>In contrast, the stellar enrichment group <tt>login_profile</tt> is interacting with the profiler, has multiple temporary expressions (i.e. <tt>profile_window</tt>, <tt>global_login_profile</tt>, and <tt>stats</tt>) that are useful only within the context of this group of stellar expressions. In this case, we would need to ensure that we use the list construct when specifying the group and remember to set the temporary variables to <tt>null</tt> so they are not passed along.</p>
+<p>In general, things to note from this section are as follows:</p>
+<ul>
+
+<li>The stellar enrichments for the <tt>stellar</tt> enrichment adapter are specified in the <tt>config</tt> for the <tt>stellar</tt> enrichment adapter in the <tt>fieldMap</tt></li>
+<li>Groups of independent (i.e. no expression in any group depend on the output of an expression from an other group) may be executed in parallel</li>
+<li>If you have the need to use temporary variables, you may use the list construct. Ensure that you assign the variables to <tt>null</tt> before the end of the group.</li>
+<li><b>Ensure that you do not assign a field to a stellar expression which returns an object which JSON cannot represent.</b></li>
+<li>Fields assigned to Maps as part of stellar enrichments have the maps unfolded, similar to the HBase Enrichment
+<ul>
+
+<li>For example the stellar enrichment for field <tt>foo</tt> which assigns a map such as <tt>foo := { 'grok' : 1, 'bar' : 'baz'}</tt> would yield the following fields:
+<ul>
+
+<li><tt>foo.grok</tt> == <tt>1</tt></li>
+<li><tt>foo.bar</tt> == <tt>'baz'</tt></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul></div></div>
+<div class="section">
+<h3><a name="The_threatIntel_Configuration"></a>The <tt>threatIntel</tt> Configuration</h3>
+<table border="0" class="table table-striped">
+<thead>
+
+<tr class="a">
+<th> Field </th>
+<th> Description </th>
+<th> Example </th></tr>
+</thead><tbody>
+
+<tr class="b">
+<td> <tt>fieldToTypeMap</tt> </td>
+<td> In the case of a simple HBase threat intel enrichment (i.e. a key/value lookup), the mapping between fields and the enrichment types associated with those fields must be known. This enrichment type is used as part of the HBase key. Note: applies to hbaseThreatIntel only. </td>
+<td> <tt>"fieldToTypeMap" : { "ip_src_addr" : [ "malicious_ips" ] }</tt> </td></tr>
+<tr class="a">
+<td> <tt>fieldMap</tt> </td>
+<td> The map of threat intel enrichment bolts names to fields in the JSON messages. Each field is sent to the threat intel enrichment bolt referenced in the key. Each field listed in the array arg is sent to the enrichment referenced in the key. Cardinality of fields to enrichments is many-to-many. </td>
+<td> <tt>"fieldMap": {"hbaseThreatIntel": ["ip_src_addr","ip_dst_addr"]}</tt> </td></tr>
+<tr class="b">
+<td> <tt>triageConfig</tt> </td>
+<td> The configuration of the threat triage scorer. In the situation where a threat is detected, a score is assigned to the message and embedded in the indexed message. </td>
+<td> <tt>"riskLevelRules" : { "IN_SUBNET(ip_dst_addr, '192.168.0.0/24')" : 10 }</tt> </td></tr>
+<tr class="a">
+<td> <tt>config</tt> </td>
+<td> The general configuration for the Threat Intel </td>
+<td> <tt>"config": {"typeToColumnFamily": { "malicious_ips","cf" } }</tt> </td></tr>
+</tbody>
+</table>
+<p>The <tt>config</tt> map is intended to house threat intel specific configuration. For instance, for the <tt>hbaseThreatIntel</tt> threat intel adapter, the mappings between the enrichment types to the column families is specified. The <tt>fieldMap</tt> configuration is similar to the <tt>enrichment</tt> configuration in that the adapters available are the same.</p>
+<p>The <tt>triageConfig</tt> field is also a complex field and it bears some description:</p>
+<table border="0" class="table table-striped">
+<thead>
+
+<tr class="a">
+<th> Field </th>
+<th> Description </th>
+<th> Example </th></tr>
+</thead><tbody>
+
+<tr class="b">
+<td> <tt>riskLevelRules</tt> </td>
+<td> This is a list of rules (represented as Stellar expressions) associated with scores with optional names and comments </td>
+<td> see below</td></tr>
+<tr class="a">
+<td> <tt>aggregator</tt> </td>
+<td> An aggregation function that takes all non-zero scores representing the matching queries from <tt>riskLevelRules</tt> and aggregates them into a single score. </td>
+<td> <tt>"MAX"</tt> </td></tr>
+</tbody>
+</table>
+<p>A risk level rule is of the following format:</p>
+<ul>
+
+<li><tt>name</tt> : The name of the threat triage rule</li>
+<li><tt>comment</tt> : A comment describing the rule</li>
+<li><tt>rule</tt> : The rule, represented as a Stellar statement</li>
+<li><tt>score</tt> : The score attributed to the rule. Can be either numeric or a Stellar expression. The expression has access to all fields with the message being triaged.</li>
+<li><tt>reason</tt> : Reason the rule tripped. Can be represented as a Stellar statement</li>
+</ul>
+<p>An example of a rule is as follows:</p>
+
+<div>
+<div>
+<pre class="source"> "riskLevelRules" : [
+ {
+ "name" : "is internal",
+ "comment" : "determines if the destination is internal.",
+ "rule" : "IN_SUBNET(ip_dst_addr, '192.168.0.0/24')",
+ "score" : 10,
+ "reason" : "FORMAT('%s is internal', ip_dst_addr)"
+ }
+ ]
+</pre></div></div>
+
+<p>The supported aggregation functions are:</p>
+<ul>
+
+<li><tt>MAX</tt> : The max of all of the associated values for matching queries</li>
+<li><tt>MIN</tt> : The min of all of the associated values for matching queries</li>
+<li><tt>MEAN</tt> : The mean of all of the associated values for matching queries</li>
+<li><tt>SUM</tt> : The sum of all the associated values for matching queries</li>
+<li><tt>POSITIVE_MEAN</tt> : The mean of the positive associated values for the matching queries.</li>
+</ul></div>
+<div class="section">
+<h3><a name="Example_Configuration"></a>Example Configuration</h3>
+<p>An example configuration for the YAF sensor is as follows:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "enrichment": {
+ "fieldMap": {
+ "geo": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ],
+ "host": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ],
+ "hbaseEnrichment": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ]
+ }
+ ,"fieldToTypeMap": {
+ "ip_src_addr": [
+ "playful_classification"
+ ],
+ "ip_dst_addr": [
+ "playful_classification"
+ ]
+ }
+ },
+ "threatIntel": {
+ "fieldMap": {
+ "hbaseThreatIntel": [
+ "ip_src_addr",
+ "ip_dst_addr"
+ ]
+ },
+ "fieldToTypeMap": {
+ "ip_src_addr": [
+ "malicious_ip"
+ ],
+ "ip_dst_addr": [
+ "malicious_ip"
+ ]
+ },
+ "triageConfig" : {
+ "riskLevelRules" : [
+ {
+ "rule" : "ip_src_addr == '10.0.2.3' or ip_dst_addr == '10.0.2.3'",
+ "score" : 10
+ }
+ ],
+ "aggregator" : "MAX"
+ }
+ }
+}
+</pre></div></div>
+
+<p>ThreatIntel alert levels are emitted as a new field “threat.triage.level.” So for the example above, an incoming message that trips the <tt>ip_src_addr</tt> rule will have a new field threat.triage.level=10.</p></div></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-enrichment/metron-enrichment-storm/Performance.html b/current-book/metron-platform/metron-enrichment/metron-enrichment-storm/Performance.html
new file mode 100644
index 0000000..ead5fc7
--- /dev/null
+++ b/current-book/metron-platform/metron-enrichment/metron-enrichment-storm/Performance.html
@@ -0,0 +1,807 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-enrichment/metron-enrichment-storm/Performance.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Enrichment Performance</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">Enrichment Performance</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-down"></span>Enrichment</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-enrichment/metron-enrichment-common/index.html" title="Enrichment-common"><span class="none"></span>Enrichment-common</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/metron-enrichment-storm/index.html" title="Enrichment-storm"><span class="icon-chevron-down"></span>Enrichment-storm</a>
+ <ul class="nav nav-list">
+ <li class="active"><a href="#"><span class="none"></span>Performance</a></li>
+ </ul>
+</li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
+ <li><a href="../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Enrichment Performance</h1>
+<p><a name="Enrichment_Performance"></a></p>
+<p>This guide defines a set of benchmarks used to measure the performance of the Enrichment topology. The guide also provides detailed steps on how to execute those benchmarks along with advice for tuning the Unified Enrichment topology.</p>
+<ul>
+
+<li><a href="#Benchmarks">Benchmarks</a></li>
+<li><a href="#Benchmark_Execution">Benchmark Execution</a></li>
+<li><a href="#Performance_Tuning">Performance Tuning</a></li>
+<li><a href="#Benchmark_Results">Benchmark Results</a></li>
+</ul>
+<div class="section">
+<h2><a name="Benchmarks"></a>Benchmarks</h2>
+<p>The following section describes a set of enrichments that will be used to benchmark the performance of the Enrichment topology.</p>
+<ul>
+
+<li><a href="#Geo_IP_Enrichment">Geo IP Enrichment</a></li>
+<li><a href="#HBase_Enrichment">HBase Enrichment</a></li>
+<li><a href="#Stellar_Enrichment">Stellar Enrichment</a></li>
+</ul>
+<div class="section">
+<h3><a name="Geo_IP_Enrichment"></a>Geo IP Enrichment</h3>
+<p>This benchmark measures the performance of executing a Geo IP enrichment. Given a valid IP address the enrichment will append detailed location information for that IP. The location information is sourced from an external Geo IP data source like <a class="externalLink" href="https://github.com/maxmind/GeoIP2-java">Maxmind</a>.</p>
+<div class="section">
+<h4><a name="Configuration"></a>Configuration</h4>
+<p>Adding the following Stellar expression to the Enrichment topology configuration will define a Geo IP enrichment.</p>
+
+<div>
+<div>
+<pre class="source">geo := GEO_GET(ip_dst_addr)
+</pre></div></div>
+
+<p>After the enrichment process completes, the telemetry message will contain a set of fields with location information for the given IP address.</p>
+
+<div>
+<div>
+<pre class="source">{
+ "ip_dst_addr":"151.101.129.140",
+ ...
+ "geo.city":"San Francisco",
+ "geo.country":"US",
+ "geo.dmaCode":"807",
+ "geo.latitude":"37.7697",
+ "geo.location_point":"37.7697,-122.3933",
+ "geo.locID":"5391959",
+ "geo.longitude":"-122.3933",
+ "geo.postalCode":"94107",
+ }
+</pre></div></div>
+</div></div>
+<div class="section">
+<h3><a name="HBase_Enrichment"></a>HBase Enrichment</h3>
+<p>This benchmark measures the performance of executing an enrichment that retrieves data from an external HBase table. This type of enrichment is useful for enriching telemetry from an Asset Database or other source of relatively static data.</p>
+<div class="section">
+<h4><a name="Configuration"></a>Configuration</h4>
+<p>Adding the following Stellar expression to the Enrichment topology configuration will define an Hbase enrichment. This looks up the ‘ip_dst_addr’ within an HBase table ‘top-1m’ and returns a hostname.</p>
+
+<div>
+<div>
+<pre class="source">top1m := ENRICHMENT_GET('top-1m', ip_dst_addr, 'top-1m', 't')
+</pre></div></div>
+
+<p>After the telemetry has been enriched, it will contain the host and IP elements that were retrieved from the HBase table.</p>
+
+<div>
+<div>
+<pre class="source">{
+ "ip_dst_addr":"151.101.2.166",
+ ...
+ "top1m.host":"earther.com",
+ "top1m.ip":"151.101.2.166"
+}
+</pre></div></div>
+</div></div>
+<div class="section">
+<h3><a name="Stellar_Enrichment"></a>Stellar Enrichment</h3>
+<p>This benchmark measures the performance of executing a basic Stellar expression. In this benchmark, the enrichment is purely a computational task that has no dependence on an external system like a database.</p>
+<div class="section">
+<h4><a name="Configuration"></a>Configuration</h4>
+<p>Adding the following Stellar expression to the Enrichment topology configuration will define a basic Stellar enrichment. The following returns true if the IP is in the given subnet and false otherwise.</p>
+
+<div>
+<div>
+<pre class="source">local := IN_SUBNET(ip_dst_addr, '192.168.0.0/24')
+</pre></div></div>
+
+<p>After the telemetry has been enriched, it will contain a field with a boolean value indicating whether the IP was within the given subnet.</p>
+
+<div>
+<div>
+<pre class="source">{
+ "ip_dst_addr":"151.101.2.166",
+ ...
+ "local":false
+}
+</pre></div></div>
+</div></div></div>
+<div class="section">
+<h2><a name="Benchmark_Execution"></a>Benchmark Execution</h2>
+<p>This section describes the steps necessary to execute the performance benchmarks for the Enrichment topology.</p>
+<ul>
+
+<li><a href="#Prepare_Enrichment_Data">Prepare Enrichment Data</a></li>
+<li><a href="#Load_HBase_with_Enrichment_Data">Load HBase with Enrichment Data</a></li>
+<li><a href="#Configure_the_Enrichments">Configure the Enrichments</a></li>
+<li><a href="#Create_Input_Telemetry">Create Input Telemetry</a></li>
+<li><a href="#Cluster_Setup">Cluster Setup</a></li>
+<li><a href="#Monitoring">Monitoring</a></li>
+</ul>
+<div class="section">
+<h3><a name="Prepare_Enrichment_Data"></a>Prepare Enrichment Data</h3>
+<p>The Alexa Top 1 Million was used as a data source for these benchmarks.</p>
+<ol style="list-style-type: decimal">
+
+<li>
+
+<p>Download the <a class="externalLink" href="http://s3.amazonaws.com/alexa-static/top-1m.csv.zip">Alexa Top 1 Million</a> or another similar data set with a variety of valid hostnames.</p>
+</li>
+<li>
+
+<p>For each hostname, query DNS to retrieve an associated IP address.</p>
+<p>A script like the following can be used for this. There is no need to do this for all 1 million entries in the data set. Doing this for around 10,000 records is sufficient.</p>
+
+<div>
+<div>
+<pre class="source">import dns.resolver
+import csv
+#
+resolver = dns.resolver.Resolver()
+resolver.nameservers = ['8.8.8.8', '8.8.4.4']
+#
+with open('top-1m.csv', 'r') as infile:
+ with open('top-1m-with-ip.csv', 'w') as outfile:
+ #
+ reader = csv.reader(infile, delimiter=',')
+ writer = csv.writer(outfile, delimiter=',')
+ for row in reader:
+ #
+ host = row[1]
+ try:
+ response = resolver.query(host, "A")
+ for record in response:
+ ip = record
+ writer.writerow([host, ip])
+ print "host={}, ip={}".format(host, ip)
+ #
+ except:
+ pass
+</pre></div></div>
+</li>
+<li>
+
+<p>The resulting data set contains an IP to hostname mapping.</p>
+
+<div>
+<div>
+<pre class="source">$ head top-1m-with-ip.csv
+google.com,172.217.9.46
+youtube.com,172.217.4.78
+facebook.com,157.240.18.35
+baidu.com,220.181.57.216
+baidu.com,111.13.101.208
+baidu.com,123.125.114.144
+wikipedia.org,208.80.154.224
+yahoo.com,98.139.180.180
+yahoo.com,206.190.39.42
+reddit.com,151.101.1.140
+</pre></div></div>
+</li>
+</ol></div>
+<div class="section">
+<h3><a name="Load_HBase_with_Enrichment_Data"></a>Load HBase with Enrichment Data</h3>
+<ol style="list-style-type: decimal">
+
+<li>
+
+<p>Create an HBase table for this data.</p>
+<p>Ensure that the table is evenly distributed across the HBase nodes. This can be done by pre-splitting the table or splitting the data after loading it.</p>
+
+<div>
+<div>
+<pre class="source">create 'top-1m', 't', {SPLITS => ['2','4','6','8','a','c','e']}
+</pre></div></div>
+</li>
+<li>
+
+<p>Create a configuration file called <tt>extractor.json</tt>. This defines how the data will be loaded into the HBase table.</p>
+
+<div>
+<div>
+<pre class="source">> cat extractor.json
+{
+ "config": {
+ "columns": {
+ "host" : 0,
+ "ip": 1
+ },
+ "indicator_column": "ip",
+ "type": "top-1m",
+ "separator": ","
+ },
+ "extractor": "CSV"
+}
+</pre></div></div>
+</li>
+<li>
+
+<p>Use the <tt>flatfile_loader.sh</tt> to load the data into the HBase table.</p>
+
+<div>
+<div>
+<pre class="source">$METRON_HOME/bin/flatfile_loader.sh \
+ -e extractor.json \
+ -t top-1m \
+ -c t \
+ -i top-1m-with-ip.csv
+</pre></div></div>
+</li>
+</ol></div>
+<div class="section">
+<h3><a name="Configure_the_Enrichments"></a>Configure the Enrichments</h3>
+<ol style="list-style-type: decimal">
+
+<li>Define the Enrichments using the REPL.
+
+<div>
+<div>
+<pre class="source">> $METRON_HOME/bin/stellar -z $ZOOKEEPER
+Stellar, Go!
+[Stellar]>>> conf
+{
+ "enrichment": {
+ "fieldMap": {
+ "stellar" : {
+ "config" : {
+ "geo" : "GEO_GET(ip_dst_addr)",
+ "top1m" : "ENRICHMENT_GET('top-1m', ip_dst_addr, 'top-1m', 't')",
+ "local" : "IN_SUBNET(ip_dst_addr, '192.168.0.0/24')"
+ }
+ }
+ },
+ "fieldToTypeMap": {
+ }
+ },
+ "threatIntel": {
+ }
+}
+[Stellar]>>> CONFIG_PUT("ENRICHMENT", conf, "asa")
+</pre></div></div>
+</li>
+</ol></div>
+<div class="section">
+<h3><a name="Create_Input_Telemetry"></a>Create Input Telemetry</h3>
+<ol style="list-style-type: decimal">
+
+<li>
+
+<p>Create a template file that defines what your input telemetry will look-like.</p>
+
+<div>
+<div>
+<pre class="source">> cat asa.template
+{"ciscotag": "ASA-1-123123", "source.type": "asa", "ip_dst_addr": "$DST_ADDR", "original_string": "<134>Feb 22 17:04:43 AHOSTNAME %ASA-1-123123: Built inbound ICMP connection for faddr 192.168.11.8/50244 gaddr 192.168.1.236/0 laddr 192.168.1.1/161", "ip_src_addr": "192.168.1.35", "syslog_facility": "local1", "action": "built", "syslog_host": "AHOSTNAME", "timestamp": "$METRON_TS", "protocol": "icmp", "guid": "$METRON_GUID", "syslog_severity": "info"}
+</pre></div></div>
+</li>
+<li>
+
+<p>Use the template file along with the enrichment data to create input telemetry with varying IP addresses.</p>
+
+<div>
+<div>
+<pre class="source">for i in $(head top-1m-with-ip.csv | awk -F, '{print $2}');do
+ cat asa.template | sed "s/\$DST_ADDR/$i/";
+done > asa.input.template
+</pre></div></div>
+</li>
+<li>
+
+<p>Use the <tt>load_tool.sh</tt> script to push messages onto the input topic <tt>enrichments</tt> and monitor the output topic <tt>indexing</tt>. See more information in the Performance <a href="metron-contrib/metron-performance/index.html">README.md</a>.</p>
+<p>If the topology is keeping up, obviously the events per second produced on the input topic should roughly match the output topic.</p>
+</li>
+</ol></div>
+<div class="section">
+<h3><a name="Cluster_Setup"></a>Cluster Setup</h3>
+<div class="section">
+<h4><a name="Isolation"></a>Isolation</h4>
+<p>The Enrichment topology depends on an environment with at least two and often three components that work together; Storm, Kafka, and HBase. When any of two of these are run on the same node, it can be difficult to identify which of them is becoming a bottleneck. This can cause poor and highly volatile performance as each steals resources from the other.</p>
+<p>It is highly recommended that each of these systems be fully isolated from the others. Storm should be run on nodes that are completely isolated from Kafka and HBase.</p></div></div>
+<div class="section">
+<h3><a name="Monitoring"></a>Monitoring</h3>
+<ol style="list-style-type: decimal">
+
+<li>
+
+<p>The <tt>load_test.sh</tt> script will report the throughput for the input and output topics.</p>
+<ul>
+
+<li>
+
+<p>The input throughput should roughly match the output throughput if the topology is able to handle a given load.</p>
+</li>
+<li>
+
+<p>Not only are the raw throughput numbers important, but also the consistency of what is reported over time. If the reported throughput is sporadic, then further tuning may be required.</p>
+</li>
+</ul>
+</li>
+<li>
+
+<p>The Storm UI is obviously an important source of information. The bolt capacity, complete latency, and any reported errors are all important to monitor</p>
+</li>
+<li>
+
+<p>The load reported by the OS is also an important metric to monitor.</p>
+<ul>
+
+<li>
+
+<p>The load metric should be monitored to ensure that each node is being pushed sufficiently, but not too much.</p>
+</li>
+<li>
+
+<p>The load should be evenly distributed across each node. If the load is uneven, this may indicate a problem.</p>
+</li>
+</ul>
+<p>A simple script like the following is sufficient for the task.</p>
+
+<div>
+<div>
+<pre class="source">for host in $(cat cluster.txt); do
+ echo $host;
+ ssh root@$host 'uptime';
+done
+</pre></div></div>
+</li>
+<li>
+
+<p>Monitoring the Kafka offset lags indicates how far behind a consumer may be. This can be very useful to determine if the topology is keeping up.</p>
+
+<div>
+<div>
+<pre class="source">${KAFKA_HOME}/bin/kafka-consumer-groups.sh \
+ --command-config=/tmp/consumergroup.config \
+ --describe \
+ --group enrichments \
+ --bootstrap-server $BROKERLIST \
+ --new-consumer
+</pre></div></div>
+</li>
+<li>
+
+<p>A tool like <a class="externalLink" href="https://github.com/yahoo/kafka-manager">Kafka Manager</a> is also very useful for monitoring the input and output topics during test execution.</p>
+</li>
+</ol></div></div>
+<div class="section">
+<h2><a name="Performance_Tuning"></a>Performance Tuning</h2>
+<p>The approach to tuning the topology will look something like the following. More detailed tuning information is available next to each named parameter</p>
+<ul>
+
+<li>
+
+<p>Start the tuning process with a single worker. After tuning the bolts within a single worker, scale out with additional worker processes.</p>
+</li>
+<li>
+
+<p>Initially set the thread pool size to 1. Increase this value slowly only after tuning the other parameters first. Consider that each worker has its own thread pool and the total size of this thread pool should be far less than the total number of cores available in the cluster.</p>
+</li>
+<li>
+
+<p>Initially set each bolt parallelism hint to the number of partitions on the input Kafka topic. Monitor bolt capacity and increase the parallelism hint for any bolt whose capacity is close to or exceeds 1.</p>
+</li>
+<li>
+
+<p>If the topology is not able to keep-up with a given input, then increasing the parallelism is the primary means to scale up.</p>
+</li>
+<li>
+
+<p>Parallelism units can be used for determining how to distribute processing tasks across the topology. The sum of parallelism can be close to, but should not far exceed this value.</p>
+<p>(number of worker nodes in cluster * number cores per worker node) - (number of acker tasks)</p>
+</li>
+<li>
+
+<p>The throughput that the topology is able to sustain should be relatively consistent. If the throughput fluctuates greatly, increase back pressure using <a href="#topology.max.spout.pending"><tt>topology.max.spout.pending</tt></a>.</p>
+</li>
+</ul>
+<div class="section">
+<h3><a name="Parameters"></a>Parameters</h3>
+<p>The following parameters are useful for tuning the “Unified” Enrichment topology.</p>
+<p>WARNING: Some of the parameter names have been reused from the “Split/Join” topology so the name may not be appropriate. This will be corrected in the future.</p>
+<ul>
+
+<li><a href="#enrichment.workers"><tt>enrichment.workers</tt></a></li>
+<li><a href="#enrichment.acker.executors"><tt>enrichment.acker.executors</tt></a></li>
+<li><a href="#topology.worker.childopts"><tt>topology.worker.childopts</tt></a></li>
+<li><a href="#topology.max.spout.pending"><tt>topology.max.spout.pending</tt></a></li>
+<li><a href="#kafka.spout.parallelism"><tt>kafka.spout.parallelism</tt></a></li>
+<li><a href="#enrichment.join.parallelism"><tt>enrichment.join.parallelism</tt></a></li>
+<li><a href="#threat.intel.join.parallelism"><tt>threat.intel.join.parallelism</tt></a></li>
+<li><a href="#kafka.writer.parallelism"><tt>kafka.writer.parallelism</tt></a></li>
+<li><a href="#enrichment.join.cache.size"><tt>enrichment.join.cache.size</tt></a></li>
+<li><a href="#threat.intel.join.cache.size"><tt>threat.intel.join.cache.size</tt></a></li>
+<li><a href="#metron.threadpool.size"><tt>metron.threadpool.size</tt></a></li>
+<li><a href="#metron.threadpool.type"><tt>metron.threadpool.type</tt></a></li>
+</ul>
+<div class="section">
+<h4><a name="enrichment.workers"></a><tt>enrichment.workers</tt></h4>
+<p>The number of worker processes for the enrichment topology.</p>
+<ul>
+
+<li>
+
+<p>Start by tuning only a single worker. Maximize throughput for that worker, then increase the number of workers.</p>
+</li>
+<li>
+
+<p>The throughput should scale relatively linearly as workers are added. This reaches a limit as the number of workers running on a single node saturate the resources available. When this happens, adding workers, but on additional nodes should allow further scaling.</p>
+</li>
+<li>
+
+<p>Increase parallelism before attempting to increase the number of workers.</p>
+</li>
+</ul></div>
+<div class="section">
+<h4><a name="enrichment.acker.executors"></a><tt>enrichment.acker.executors</tt></h4>
+<p>The number of ackers within the topology.</p>
+<ul>
+
+<li>
+
+<p>This should most often be equal to the number of workers defined in <tt>enrichment.workers</tt>.</p>
+</li>
+<li>
+
+<p>Within the Storm UI, click the “Show System Stats” button. This will display a bolt named <tt>__acker</tt>. If the capacity of this bolt is too high, then increase the number of ackers.</p>
+</li>
+</ul></div>
+<div class="section">
+<h4><a name="topology.worker.childopts"></a><tt>topology.worker.childopts</tt></h4>
+<p>This parameter accepts arguments that will be passed to the JVM created for each Storm worker. This allows for control over the heap size, garbage collection, and any other JVM-specific parameter.</p>
+<ul>
+
+<li>
+
+<p>Start with a 2G heap and increase as needed. Running with 8G was found to be beneficial, but will vary depending on caching needs.</p>
+<p><tt>-Xms8g -Xmx8g</tt></p>
+</li>
+<li>
+
+<p>The Garbage First Garbage Collector (G1GC) is recommended along with a cap on the amount of time spent in garbage collection. This is intended to help address small object allocation issues due to our extensive use of caches.</p>
+<p><tt>-XX:+UseG1GC -XX:MaxGCPauseMillis=100</tt></p>
+</li>
+<li>
+
+<p>If the caches in use are very large (as defined by either <a href="#enrichment.join.cache.size"><tt>enrichment.join.cache.size</tt></a> or <a href="#threat.intel.join.cache.size"><tt>threat.intel.join.cache.size</tt></a>) and performance is poor, turning on garbage collection logging might be helpful.</p>
+</li>
+</ul></div>
+<div class="section">
+<h4><a name="topology.max.spout.pending"></a><tt>topology.max.spout.pending</tt></h4>
+<p>This limits the number of unacked tuples that the spout can introduce into the topology.</p>
+<ul>
+
+<li>
+
+<p>Decreasing this value will increase back pressure and allow the topology to consume messages at a pace that is maintainable.</p>
+</li>
+<li>
+
+<p>If the spout throws ‘Commit Failed Exceptions’ then the topology is not keeping up. Decreasing this value is one way to ensure that messages can be processed before they time out.</p>
+</li>
+<li>
+
+<p>If the topology’s throughput is unsteady and inconsistent, decrease this value. This should help the topology consume messages at a maintainable pace.</p>
+</li>
+<li>
+
+<p>If the bolt capacity is low, the topology can handle additional load. Increase this value so that more tuples are introduced into the topology which should increase the bolt capacity.</p>
+</li>
+</ul></div>
+<div class="section">
+<h4><a name="kafka.spout.parallelism"></a><tt>kafka.spout.parallelism</tt></h4>
+<p>The parallelism of the Kafka spout within the topology. Defines the maximum number of executors for each worker dedicated to running the spout.</p>
+<ul>
+
+<li>
+
+<p>The spout parallelism should most often be set to the number of partitions of the input Kafka topic.</p>
+</li>
+<li>
+
+<p>If the enrichment bolt capacity is low, increasing the parallelism of the spout can introduce additional load on the topology.</p>
+</li>
+</ul></div>
+<div class="section">
+<h4><a name="enrichment.join.parallelism"></a><tt>enrichment.join.parallelism</tt></h4>
+<p>The parallelism hint for the enrichment bolt. Defines the maximum number of executors within each worker dedicated to running the enrichment bolt.</p>
+<p>WARNING: The property name does not match its current usage in the Unified topology. This property name may change in the near future as it has been reused from the Split-Join topology.</p>
+<ul>
+
+<li>
+
+<p>If the capacity of the enrichment bolt is high, increasing the parallelism will introduce additional executors to bring the bolt capacity down.</p>
+</li>
+<li>
+
+<p>If the throughput of the topology is too low, increase this value. This allows additional tuples to be enriched in parallel.</p>
+</li>
+<li>
+
+<p>Increasing parallelism on the enrichment bolt will at some point put pressure on the downstream threat intel and output bolts. As this value is increased, monitor the capacity of the downstream bolts to ensure that they do not become a bottleneck.</p>
+</li>
+</ul></div>
+<div class="section">
+<h4><a name="threat.intel.join.parallelism"></a><tt>threat.intel.join.parallelism</tt></h4>
+<p>The parallelism hint for the threat intel bolt. Defines the maximum number of executors within each worker dedicated to running the threat intel bolt.</p>
+<p>WARNING: The property name does not match its current usage in the Unified topology. This property name may change in the near future as it has been reused from the Split-Join topology.</p>
+<ul>
+
+<li>
+
+<p>If the capacity of the threat intel bolt is high, increasing the parallelism will introduce additional executors to bring the bolt capacity down.</p>
+</li>
+<li>
+
+<p>If the throughput of the topology is too low, increase this value. This allows additional tuples to be enriched in parallel.</p>
+</li>
+<li>
+
+<p>Increasing parallelism on this bolt will at some point put pressure on the downstream output bolt. As this value is increased, monitor the capacity of the output bolt to ensure that it does not become a bottleneck.</p>
+</li>
+</ul></div>
+<div class="section">
+<h4><a name="kafka.writer.parallelism"></a><tt>kafka.writer.parallelism</tt></h4>
+<p>The parallelism hint for the output bolt which writes to the output Kafka topic. Defines the maximum number of executors within each worker dedicated to running the output bolt.</p>
+<ul>
+
+<li>If the capacity of the output bolt is high, increasing the parallelism will introduce additional executors to bring the bolt capacity down.</li>
+</ul></div>
+<div class="section">
+<h4><a name="enrichment.join.cache.size"></a><tt>enrichment.join.cache.size</tt></h4>
+<p>The Enrichment bolt maintains a cache so that if the same enrichment occurs repetitively, the value can be retrieved from the cache instead of it being recomputed.</p>
+<p>There is a great deal of repetition in network telemetry, which leads to a great deal of repetition for the enrichments that operate on that telemetry. Having a highly performant cache is one of the most critical factors driving performance.</p>
+<p>WARNING: The property name does not match its current usage in the Unified topology. This property name may change in the near future as it has been reused from the Split-Join topology.</p>
+<ul>
+
+<li>
+
+<p>Increase the size of the cache to improve the rate of cache hits.</p>
+</li>
+<li>
+
+<p>Increasing the size of the cache may require that you increase the worker heap size using `topology.worker.childopts’.</p>
+</li>
+</ul></div>
+<div class="section">
+<h4><a name="threat.intel.join.cache.size"></a><tt>threat.intel.join.cache.size</tt></h4>
+<p>The Threat Intel bolt maintains a cache so that if the same enrichment occurs repetitively, the value can be retrieved from the cache instead of it being recomputed.</p>
+<p>There is a great deal of repetition in network telemetry, which leads to a great deal of repetition for the enrichments that operate on that telemetry. Having a highly performant cache is one of the most critical factors driving performance.</p>
+<p>WARNING: The property name does not match its current usage in the Unified topology. This property name may change in the near future as it has been reused from the Split-Join topology.</p>
+<ul>
+
+<li>
+
+<p>Increase the size of the cache to improve the rate of cache hits.</p>
+</li>
+<li>
+
+<p>Increasing the size of the cache may require that you increase the worker heap size using `topology.worker.childopts’.</p>
+</li>
+</ul></div>
+<div class="section">
+<h4><a name="metron.threadpool.size"></a><tt>metron.threadpool.size</tt></h4>
+<p>This value defines the number of threads maintained within a pool to execute each enrichment. This value can either be a fixed number or it can be a multiple of the number of cores (5C = 5 times the number of cores).</p>
+<p>The enrichment bolt maintains a static thread pool that is used to execute each enrichment. This thread pool is shared by all of the executors running within the same worker.</p>
+<p>WARNING: This value must be manually defined within the flux file at <tt>$METRON_HOME/flux/enrichment/remote-unified.yaml</tt>. This value cannot be altered within Ambari at this time.</p>
+<ul>
+
+<li>
+
+<p>Start with a thread pool size of 1. Adjust this value after tuning all other parameters first. Only increase this value if testing shows performance improvements in your environment given your workload.</p>
+</li>
+<li>
+
+<p>If the thread pool size is too large this will cause the work to be shuffled amongst multiple CPU cores, which significantly decreases performance. Using a smaller thread pool helps pin work to a single core.</p>
+</li>
+<li>
+
+<p>If the thread pool size is too small this can negatively impact IO-intensive workloads. Increasing the thread pool size, helps when using IO-intensive workloads with a significant cache miss rate. A thread pool size of 3-5 can help in these cases.</p>
+</li>
+<li>
+
+<p>Most workloads will make significant use of the cache and so 1-2 threads will most likely be optimal.</p>
+</li>
+<li>
+
+<p>The bolt uses a static thread pool. To scale out, but keep the work mostly pinned to a CPU core, add more Storm workers while keeping the thread pool size low.</p>
+</li>
+<li>
+
+<p>If a larger thread pool increases load on the system, but decreases the throughput, then it is likely that the system is thrashing. In this case the thread pool size should be decreased.</p>
+</li>
+</ul></div>
+<div class="section">
+<h4><a name="metron.threadpool.type"></a><tt>metron.threadpool.type</tt></h4>
+<p>The enrichment bolt maintains a static thread pool that is used to execute each enrichment. This thread pool is shared by all of the executors running within the same worker.</p>
+<p>Defines the type of thread pool used. This value can be either “FIXED” or “WORK_STEALING”.</p>
+<p>Currently, this value must be manually defined within the flux file at <tt>$METRON_HOME/flux/enrichment/remote-unified.yaml</tt>. This value cannot be altered within Ambari.</p></div></div>
+<div class="section">
+<h3><a name="Benchmark_Results"></a>Benchmark Results</h3>
+<p>This section describes one execution of these benchmarks to help provide an understanding of what reasonably tuned parameters might look-like.</p>
+<p>These parameters and the throughput reported are highly dependent on the workload and resources available. The throughput is what was achievable given a reasonable amount of tuning on a small, dedicated cluster. The throughput is largely dependent on the enrichments performed and the distribution of data within the incoming telemetry.</p>
+<p>The Enrichment topology has been show to scale relatively linearly. Adding more resources allows for more complex enrichments, across more diverse data sets, at higher volumes. The throughput that one might see in production largely depends on how much hardware can be committed to the task.</p>
+<div class="section">
+<h4><a name="Environment"></a>Environment</h4>
+<ul>
+
+<li>
+
+<p>Apache Metron 0.5.0 (pre-release) March, 2018</p>
+<ul>
+
+<li>This included <a class="externalLink" href="https://github.com/apache/metron/pull/947">a patch to the underlying caching mechanism</a> that greatly improves performance.</li>
+</ul>
+</li>
+<li>
+
+<p>Cisco UCS nodes</p>
+<ul>
+
+<li>32 core, 64-bit CPU (Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz)</li>
+<li>256 GB RAM</li>
+<li>x2 10G NIC bonded</li>
+<li>x4 6TB 7200 RPM disks</li>
+</ul>
+</li>
+<li>
+
+<p>Storm Supervisors are isolated and running on a dedicated set of 3 nodes.</p>
+</li>
+<li>
+
+<p>Kafka Brokers are isolated and running on a separate, dedicated set of 3 nodes.</p>
+</li>
+</ul></div>
+<div class="section">
+<h4><a name="Results"></a>Results</h4>
+<ul>
+
+<li>
+
+<p>These benchmarks executed all 3 enrichments simultaneously; the <a href="#Geo_IP_Enrichment">Geo IP Enrichment</a>, <a href="#Stellar_Enrichment">Stellar Enrichment</a> and the <a href="#HBase_Enrichment">HBase Enrichment</a>.</p>
+</li>
+<li>
+
+<p>The data used to drive the benchmark includes 10,000 unique IP addresses. The telemetry was populated with IP addresses such that 10% of these IPs were chosen 80% of the time. This bias was designed to mimic the typical distribution seen in real-world telemetry.</p>
+</li>
+<li>
+
+<p>The Unified Enrichment topology was able to sustain 308,000 events per second on a small, dedicated 3 node cluster.</p>
+</li>
+<li>
+
+<p>The values used to achieve these results with the Unified Enrichment topology follows. You should not attempt to use these parameters in your topology directly. These are specific to the environment and workload and should only be used as a guideline.</p>
+
+<div>
+<div>
+<pre class="source">enrichment.workers=9
+enrichment.acker.executors=9
+enrichment.join.cache.size=100000
+threat.intel.join.cache.size=100000
+kafka.spout.parallelism=27
+enrichment.join.parallelism=54
+threat.intel.join.parallelism=9
+kafka.writer.parallelism=27
+topology.worker.childopts=-XX:+UseG1GC -Xms8g -Xmx8g -XX:MaxGCPauseMillis=100
+topology.max.spout.pending=3000
+metron.threadpool.size=1
+metron.threadpool.type=FIXED
+</pre></div></div>
+</li>
+</ul></div></div></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-enrichment/metron-enrichment-storm/index.html b/current-book/metron-platform/metron-enrichment/metron-enrichment-storm/index.html
new file mode 100644
index 0000000..3becc42
--- /dev/null
+++ b/current-book/metron-platform/metron-enrichment/metron-enrichment-storm/index.html
@@ -0,0 +1,261 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-enrichment/metron-enrichment-storm/index.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Enrichment</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">Enrichment</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-down"></span>Enrichment</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-enrichment/metron-enrichment-common/index.html" title="Enrichment-common"><span class="none"></span>Enrichment-common</a></li>
+ <li class="active"><a href="#"><span class="icon-chevron-down"></span>Enrichment-storm</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-enrichment/metron-enrichment-storm/Performance.html" title="Performance"><span class="none"></span>Performance</a></li>
+ </ul>
+</li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
+ <li><a href="../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Enrichment</h1>
+<p><a name="Enrichment"></a></p>
+<div class="section">
+<h2><a name="Introduction"></a>Introduction</h2>
+<p>This module holds code for enrichments running in a Storm topology.</p></div>
+<div class="section">
+<h2><a name="Enrichment_Architecture"></a>Enrichment Architecture</h2>
+<p><img src="../../../images/unified_enrichment_arch.svg" alt="Unified Architecture" /></p>
+<div class="section">
+<h3><a name="Unified_Enrichment_Topology"></a>Unified Enrichment Topology</h3>
+<p>The unified enrichment topology uses data parallelism as opposed to the deprecated split/join topology’s task parallelism. This architecture uses a worker pool to fully enrich any message within a worker. This results in</p>
+<ul>
+
+<li>Fewer bolts in the topology</li>
+<li>Each bolt fully operates on a message.</li>
+<li>Fewer network hops</li>
+</ul>
+<p>This architecture is fully backwards compatible with the old split-join topology; the only difference is how the enrichment will operate on each message (in one bolt where the split/join is done in a threadpool as opposed to split across multiple bolts).</p>
+<div class="section">
+<h4><a name="Configuring_It"></a>Configuring It</h4>
+<p>There are two parameters which you might want to tune in this topology. Both of them are topology configuration adjustable in the flux file <tt>$METRON_HOME/config/flux/enrichment/remote-unified.yaml</tt>:</p>
+<ul>
+
+<li><tt>metron.threadpool.size</tt> : The size of the threadpool. This can take a number or a multiple of the number of cores (e.g. <tt>5C</tt> to 5 times the number of cores). The default is <tt>2C</tt>.</li>
+<li><tt>metron.threadpool.type</tt> : The type of threadpool. (note: descriptions taken from <a class="externalLink" href="https://zeroturnaround.com/rebellabs/fixedthreadpool-cachedthreadpool-or-forkjoinpool-picking-correct-java-executors-for-background-tasks/">here</a>).
+<ul>
+
+<li><tt>FIXED</tt> is a fixed threadpool of size <tt>n</tt>. <tt>n</tt> threads will process tasks at the time, when the pool is saturated, new tasks will get added to a queue without a limit on size. Good for CPU intensive tasks. This is the default.</li>
+<li><tt>WORK_STEALING</tt> is a work stealing threadpool. This will create and shut down threads dynamically to accommodate the required parallelism level. It also tries to reduce the contention on the task queue, so can be really good in heavily loaded environments. Also good when your tasks create more tasks for the executor, like recursive tasks.</li>
+</ul>
+</li>
+</ul>
+<p>In order to configure the parallelism for the enrichment bolt and threat intel bolt, the configurations will be taken from the respective join bolt parallelism. When proper ambari support for this is added, we will add its own property.</p></div></div>
+<div class="section">
+<h3><a name="Split-Join_Enrichment_Topology"></a>Split-Join Enrichment Topology</h3>
+<p>The now-deprecated split/join topology is also available and performs enrichments in parallel. This poses some issues in terms of ease of tuning and reasoning about performance.</p>
+<p><img src="../../../images/enrichment_arch.png" alt="Architecture" /></p>
+<div class="section">
+<h4><a name="Using_It"></a>Using It</h4>
+<p>In order to use the older, deprecated topology, you will need to</p>
+<ul>
+
+<li>Edit <tt>$METRON_HOME/bin/start_enrichment_topology.sh</tt> and adjust it to use <tt>remote-splitjoin.yaml</tt> instead of <tt>remote-unified.yaml</tt></li>
+<li>Restart the enrichment topology.</li>
+</ul></div></div></div>
+<div class="section">
+<h2><a name="Enrichment_Configuration"></a>Enrichment Configuration</h2>
+<p>The configuration for the <tt>enrichment</tt> topology, the topology primarily responsible for enrichment and threat intelligence enrichment, is defined by JSON documents stored in zookeeper.</p>
+<p>See <a href="../metron-enrichment-common/index.html#Enrichment_Configuration">Enrichment Configuration</a></p>
+<p><a name="Example_Enrichment_via_Stellar"></a></p>
+<h1>Example Enrichment via Stellar</h1>
+<p>Let’s walk through doing a simple enrichment using Stellar on your cluster using the Squid topology.</p></div>
+<div class="section">
+<h2><a name="Install_Prerequisites"></a>Install Prerequisites</h2>
+<p>Now let’s install some prerequisites:</p>
+<ul>
+
+<li>Squid client via <tt>yum install squid</tt></li>
+<li>ES Head plugin via <tt>/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head</tt></li>
+</ul>
+<p>Start Squid via <tt>service squid start</tt></p></div>
+<div class="section">
+<h2><a name="Adjust_Enrichment_Configurations_for_Squid_to_Call_Stellar"></a>Adjust Enrichment Configurations for Squid to Call Stellar</h2>
+<p>Let’s adjust the configurations for the Squid topology to annotate the messages using some Stellar functions.</p>
+<ul>
+
+<li>Edit the squid enrichment configuration at <tt>$METRON_HOME/config/zookeeper/enrichments/squid.json</tt> (this file will not exist, so create a new one) to add some new fields based on stellar queries:</li>
+</ul>
+
+<div>
+<div>
+<pre class="source">{
+ "enrichment" : {
+ "fieldMap": {
+ "stellar" : {
+ "config" : {
+ "numeric" : {
+ "foo": "1 + 1"
+ }
+ ,"ALL_CAPS" : "TO_UPPER(source.type)"
+ }
+ }
+ }
+ },
+ "threatIntel" : {
+ "fieldMap":{
+ "stellar" : {
+ "config" : {
+ "bar" : "TO_UPPER(source.type)"
+ }
+ }
+ },
+ "triageConfig" : {
+ }
+ }
+}
+</pre></div></div>
+
+<p>We have added the following fields as part of the enrichment phase of the enrichment topology:</p>
+<ul>
+
+<li><tt>foo</tt> == 2</li>
+<li><tt>ALL_CAPS</tt> == SQUID</li>
+</ul>
+<p>We have added the following as part of the threat intel:</p>
+<ul>
+
+<li><tt>bar</tt> == SQUID</li>
+</ul>
+<p>Please note that foo and ALL_CAPS will be applied in separate workers due to them being in separate groups.</p>
+<ul>
+
+<li>Upload new configs via <tt>$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -i $METRON_HOME/config/zookeeper -z node1:2181</tt></li>
+<li>Make the Squid topic in kafka via <tt>/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper node1:2181 --create --topic squid --partitions 1 --replication-factor 1</tt></li>
+</ul></div>
+<div class="section">
+<h2><a name="Start_Topologies_and_Send_Data"></a>Start Topologies and Send Data</h2>
+<p>Now we need to start the topologies and send some data:</p>
+<ul>
+
+<li>Start the squid topology via <tt>$METRON_HOME/bin/start_parser_topology.sh -k node1:6667 -z node1:2181 -s squid</tt></li>
+<li>Generate some data via the squid client:
+<ul>
+
+<li><tt>squidclient http://yahoo.com</tt></li>
+<li><tt>squidclient http://cnn.com</tt></li>
+</ul>
+</li>
+<li>Send the data to kafka via <tt>cat /var/log/squid/access.log | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic squid</tt></li>
+<li>Browse the data in elasticsearch via the ES Head plugin @ <a class="externalLink" href="http://node1:9200/_plugin/head/">http://node1:9200/_plugin/head/</a> and verify that in the squid index you have two documents</li>
+<li>Ensure that the documents have new fields <tt>foo</tt>, <tt>bar</tt> and <tt>ALL_CAPS</tt> with values as described above.</li>
+</ul>
+<p>Note that we could have used any Stellar statements here, including calling out to HBase via <tt>ENRICHMENT_GET</tt> and <tt>ENRICHMENT_EXISTS</tt> or even calling a machine learning model via <a href="../../../metron-analytics/metron-maas-service/index.html">Model as a Service</a>.</p></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-hbase-server/index.html b/current-book/metron-platform/metron-hbase-server/index.html
new file mode 100644
index 0000000..a378f36
--- /dev/null
+++ b/current-book/metron-platform/metron-hbase-server/index.html
@@ -0,0 +1,140 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-hbase-server/index.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – HBase Server</title>
+ <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../css/site.css" />
+ <link rel="stylesheet" href="../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">HBase Server</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li class="active"><a href="#"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
+ <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>HBase Server</h1>
+<p><a name="HBase_Server"></a></p>
+<p><a name="Introduction"></a></p>
+<h1>Introduction</h1>
+<p>This project provides HBase server-side code such as coprocessors.</p>
+<p><a name="Coprocessors"></a></p>
+<h1>Coprocessors</h1>
+<div class="section">
+<h2><a name="Enrichment_Coprocessor"></a>Enrichment Coprocessor</h2>
+<div class="section">
+<h3><a name="Properties"></a>Properties</h3>
+<p>Below is the list of properties to configure the HBase enrichment coprocessor <tt>org.apache.metron.hbase.coprocessor.EnrichmentCoprocessor</tt> for writing to HBase. These configuration properties all pulled from the global config.</p>
+<div class="section">
+<h4><a name="enrichment.list.hhase.provider.impl"></a><tt>enrichment.list.hhase.provider.impl</tt></h4>
+<p>Provider to use for obtaining the HBase table. This class implementation implements <tt>TableProvider</tt> and provides access to an <tt>HTableInterface</tt>. Defaults to <tt>org.apache.metron.hbase.HTableProvider</tt>.</p></div>
+<div class="section">
+<h4><a name="enrichment.list.hbase.table"></a><tt>enrichment.list.hbase.table</tt></h4>
+<p>HBase table name for the enrichments list. Defaults to <tt>enrichment_list</tt>.</p></div>
+<div class="section">
+<h4><a name="enrichment.list.hbase.cf"></a><tt>enrichment.list.hbase.cf</tt></h4>
+<p>HBase table column family for the enrichments list. Defaults to <tt>t</tt>.</p></div></div></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-indexing/index.html b/current-book/metron-platform/metron-indexing/index.html
index 5763be7..dc0ba67 100644
--- a/current-book/metron-platform/metron-indexing/index.html
+++ b/current-book/metron-platform/metron-indexing/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-indexing/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-indexing/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Indexing</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Indexing</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -59,10 +57,11 @@
<li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
<li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
<li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li class="active"><a href="#"><span class="none"></span>Indexing</a></li>
<li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
<li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
- <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
<li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
<li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
diff --git a/current-book/metron-platform/metron-job/index.html b/current-book/metron-platform/metron-job/index.html
index 0fb4b7d..c402a50 100644
--- a/current-book/metron-platform/metron-job/index.html
+++ b/current-book/metron-platform/metron-job/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-job/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-job/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Job</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Job</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -59,10 +57,11 @@
<li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
<li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
<li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
<li class="active"><a href="#"><span class="none"></span>Job</a></li>
<li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
- <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
<li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
<li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
diff --git a/current-book/metron-platform/metron-management/index.html b/current-book/metron-platform/metron-management/index.html
index ce18f57..9e1a750 100644
--- a/current-book/metron-platform/metron-management/index.html
+++ b/current-book/metron-platform/metron-management/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-management/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-management/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Stellar REPL Management Utilities</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Stellar REPL Management Utilities</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -59,10 +57,11 @@
<li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
<li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
<li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
<li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
<li class="active"><a href="#"><span class="none"></span>Management</a></li>
- <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
<li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
<li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
@@ -141,7 +140,6 @@
<li><a href="#Manage_Stellar_Field_Transformations">Manage Stellar Field Transformations</a></li>
<li><a href="#Manage_Stellar_Enrichments">Manage Stellar Enrichments</a></li>
<li><a href="#Manage_Threat_Triage_Rules">Manage Threat Triage Rules</a></li>
-<li><a href="#Simulate_Threat_Triage_Rules">Simulate Threat Triage Rules</a></li>
</ul>
</li>
</ul>
@@ -155,14 +153,111 @@
<li>File functions - Functions around interacting with local or HDFS files</li>
<li>Configuration functions - Functions surrounding pulling and pushing configs from zookeeper</li>
<li>Parser functions - Functions surrounding adding, viewing, and removing Parser functions.</li>
+<li>Indexing functions - Functions surrounding setting indexing parameters (batch size and timeout), setting the index, enabling/disabling index writer for sensors.</li>
<li>Enrichment functions - Functions surrounding adding, viewing and removing Stellar enrichments as well as managing batch size, batch timeout, and index names for the enrichment topology configuration</li>
<li>Threat Triage functions - Functions surrounding adding, viewing and removing threat triage functions.</li>
</ul>
+<table border="0" class="table table-striped">
+<thead>
+
+<tr class="a">
+<th align="center"> Type </th>
+<th align="center"> Function Name </th></tr>
+</thead><tbody>
+
+<tr class="b">
+<td align="center"> <b>Grok Functions</b> </td>
+<td align="center"> <a href="#GROK_EVAL"><tt>GROK_EVAL</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#GROK_PREDICT"><tt>GROK_PREDICT</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> <b>File Functions (Local)</b> </td>
+<td align="center"> <a href="#LOCAL_LS"><tt>LOCAL_LS</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#LOCAL_RM"><tt>LOCAL_RM</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> </td>
+<td align="center"> <a href="#LOCAL_READ"><tt>LOCAL_READ</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> <b>File Functions (HDFS)</b> </td>
+<td align="center"> <a href="#HDFS_LS"><tt>HDFS_LS</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> </td>
+<td align="center"> <a href="#HDFS_RM"><tt>HDFS_RM</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#HDFS_READ"><tt>HDFS_READ</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> <b>Configuration Functions</b> </td>
+<td align="center"> <a href="#CONFIG_GET"><tt>CONFIG_GET</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#CONFIG_PUT"><tt>CONFIG_PUT</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> <b>Parser Functions</b> </td>
+<td align="center"> <a href="#PARSER_CONFIG"><tt>PARSER_CONFIG</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#PARSER_INIT"><tt>PARSER_INIT</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> </td>
+<td align="center"> <a href="#PARSER_PARSE"><tt>PARSER_PARSE</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#PARSER_STELLAR_TRANSFORM_ADD"><tt>PARSER_STELLAR_TRANSFORM_ADD</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> </td>
+<td align="center"> <a href="#PARSER_STELLAR_TRANSFORM_PRINT"><tt>PARSER_STELLAR_TRANSFORM_PRINT</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#PARSER_STELLAR_TRANSFORM_REMOVE"><tt>PARSER_STELLAR_TRANSFORM_REMOVE</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> <b>Threat Triage Functions</b> </td>
+<td align="center"> <a href="#THREAT_TRIAGE_INIT"><tt>THREAT_TRIAGE_INIT</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#THREAT_TRIAGE_CONFIG"><tt>THREAT_TRIAGE_CONFIG</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> </td>
+<td align="center"> <a href="#THREAT_TRIAGE_SCORE"><tt>THREAT_TRIAGE_SCORE</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#THREAT_TRIAGE_ADD"><tt>THREAT_TRIAGE_ADD</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> </td>
+<td align="center"> <a href="#THREAT_TRIAGE_REMOVE"><tt>THREAT_TRIAGE_REMOVE</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#THREAT_TRIAGE_PRINT"><tt>THREAT_TRIAGE_PRINT</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> </td>
+<td align="center"> <a href="#THREAT_TRIAGE_SET_AGGREGATOR"><tt>THREAT_TRIAGE_SET_AGGREGATOR</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> <b>Indexing Functions</b> </td>
+<td align="center"> <a href="#INDEXING_SET_BATCH"><tt>INDEXING_SET_BATCH</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> </td>
+<td align="center"> <a href="#INDEXING_SET_ENABLED"><tt>INDEXING_SET_ENABLED</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#INDEXING_SET_INDEX"><tt>INDEXING_SET_INDEX</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> <b>Enrichment Functions</b> </td>
+<td align="center"> <a href="#ENRICHMENT_STELLAR_TRANSFORM_ADD"><tt>ENRICHMENT_STELLAR_TRANSFORM_ADD</tt></a> </td></tr>
+<tr class="a">
+<td align="center"> </td>
+<td align="center"> <a href="#ENRICHMENT_STELLAR_TRANSFORM_PRINT"><tt>ENRICHMENT_STELLAR_TRANSFORM_PRINT</tt></a> </td></tr>
+<tr class="b">
+<td align="center"> </td>
+<td align="center"> <a href="#ENRICHMENT_STELLAR_TRANSFORM_REMOVE"><tt>ENRICHMENT_STELLAR_TRANSFORM_REMOVE</tt></a> </td></tr>
+</tbody>
+</table>
<div class="section">
<h3><a name="Grok_Functions"></a>Grok Functions</h3>
-<ul>
-
-<li><tt>GROK_EVAL</tt>
+<div class="section">
+<h4><a name="GROK_EVAL"></a><tt>GROK_EVAL</tt></h4>
<ul>
<li>Description: Evaluate a grok expression for a statement.</li>
@@ -174,9 +269,9 @@
</ul>
</li>
<li>Returns: The Map associated with the grok expression being evaluated on the list of messages.</li>
-</ul>
-</li>
-<li><tt>GROK_PREDICT</tt>
+</ul></div>
+<div class="section">
+<h4><a name="GROK_PREDICT"></a><tt>GROK_PREDICT</tt></h4>
<ul>
<li>Description: Discover a grok statement for an input doc</li>
@@ -187,17 +282,13 @@
</ul>
</li>
<li>Returns: A grok expression that should match the data.</li>
-</ul>
-</li>
-</ul></div>
+</ul></div></div>
<div class="section">
<h3><a name="File_Functions"></a>File Functions</h3>
-<ul>
-
-<li>Local Files
-<ul>
-
-<li><tt>LOCAL_LS</tt>
+<div class="section">
+<h4><a name="Local_Files"></a>Local Files</h4>
+<div class="section">
+<h5><a name="LOCAL_LS"></a><tt>LOCAL_LS</tt></h5>
<ul>
<li>Description: Lists the contents of a directory.</li>
@@ -208,9 +299,9 @@
</ul>
</li>
<li>Returns: The contents of the directory in tabular form sorted by last modification date.</li>
-</ul>
-</li>
-<li><tt>LOCAL_RM</tt>
+</ul></div>
+<div class="section">
+<h5><a name="LOCAL_RM"></a><tt>LOCAL_RM</tt></h5>
<ul>
<li>Description: Removes the path</li>
@@ -222,9 +313,9 @@
</ul>
</li>
<li>Returns: boolean - true if successful, false otherwise</li>
-</ul>
-</li>
-<li><tt>LOCAL_READ</tt>
+</ul></div>
+<div class="section">
+<h5><a name="LOCAL_READ"></a><tt>LOCAL_READ</tt></h5>
<ul>
<li>Description: Retrieves the contents as a string of a file.</li>
@@ -235,9 +326,9 @@
</ul>
</li>
<li>Returns: The contents of the file and null otherwise.</li>
-</ul>
-</li>
-<li><tt>LOCAL_READ_LINES</tt>
+</ul></div>
+<div class="section">
+<h5><a name="LOCAL_READ_LINES"></a><tt>LOCAL_READ_LINES</tt></h5>
<ul>
<li>Description: Retrieves the contents of a file as a list of strings.</li>
@@ -248,9 +339,9 @@
</ul>
</li>
<li>Returns: A list of lines</li>
-</ul>
-</li>
-<li><tt>LOCAL_WRITE</tt>
+</ul></div>
+<div class="section">
+<h5><a name="LOCAL_WRITE"></a><tt>LOCAL_WRITE</tt></h5>
<ul>
<li>Description: Writes the contents of a string to a local file</li>
@@ -262,14 +353,11 @@
</ul>
</li>
<li>Returns: true if the file was written and false otherwise.</li>
-</ul>
-</li>
-</ul>
-</li>
-<li>HDFS Files
-<ul>
-
-<li><tt>HDFS_LS</tt>
+</ul></div></div>
+<div class="section">
+<h4><a name="HDFS_Files"></a>HDFS Files</h4>
+<div class="section">
+<h5><a name="HDFS_LS"></a><tt>HDFS_LS</tt></h5>
<ul>
<li>Description: Lists the contents of a directory in HDFS.</li>
@@ -280,9 +368,9 @@
</ul>
</li>
<li>Returns: The contents of the directory in tabular form sorted by last modification date.</li>
-</ul>
-</li>
-<li><tt>HDFS_RM</tt>
+</ul></div>
+<div class="section">
+<h5><a name="HDFS_RM"></a><tt>HDFS_RM</tt></h5>
<ul>
<li>Description: Removes the path in HDFS.</li>
@@ -294,9 +382,9 @@
</ul>
</li>
<li>Returns: boolean - true if successful, false otherwise</li>
-</ul>
-</li>
-<li><tt>HDFS_READ</tt>
+</ul></div>
+<div class="section">
+<h5><a name="HDFS_READ"></a><tt>HDFS_READ</tt></h5>
<ul>
<li>Description: Retrieves the contents as a string of a file in HDFS.</li>
@@ -307,9 +395,9 @@
</ul>
</li>
<li>Returns: The contents of the file and null otherwise.</li>
-</ul>
-</li>
-<li><tt>HDFS_READ_LINES</tt>
+</ul></div>
+<div class="section">
+<h5><a name="HDFS_READ_LINES"></a><tt>HDFS_READ_LINES</tt></h5>
<ul>
<li>Description: Retrieves the contents of a HDFS file as a list of strings.</li>
@@ -320,9 +408,9 @@
</ul>
</li>
<li>Returns: A list of lines</li>
-</ul>
-</li>
-<li><tt>HDFS_WRITE</tt>
+</ul></div>
+<div class="section">
+<h5><a name="HDFS_WRITE"></a><tt>HDFS_WRITE</tt></h5>
<ul>
<li>Description: Writes the contents of a string to a HDFS file</li>
@@ -334,16 +422,11 @@
</ul>
</li>
<li>Returns: true if the file was written and false otherwise.</li>
-</ul>
-</li>
-</ul>
-</li>
-</ul></div>
+</ul></div></div></div>
<div class="section">
<h3><a name="Configuration_Functions"></a>Configuration Functions</h3>
-<ul>
-
-<li><tt>CONFIG_GET</tt>
+<div class="section">
+<h4><a name="CONFIG_GET"></a><tt>CONFIG_GET</tt></h4>
<ul>
<li>Description: Retrieve a Metron configuration from zookeeper.</li>
@@ -356,9 +439,9 @@
</ul>
</li>
<li>Returns: The String representation of the config in zookeeper</li>
-</ul>
-</li>
-<li><tt>CONFIG_PUT</tt>
+</ul></div>
+<div class="section">
+<h4><a name="CONFIG_PUT"></a><tt>CONFIG_PUT</tt></h4>
<ul>
<li>Description: Updates a Metron config to Zookeeper.</li>
@@ -371,14 +454,243 @@
</ul>
</li>
<li>Returns: The String representation of the config in zookeeper</li>
-</ul>
-</li>
-</ul></div>
+</ul></div></div>
<div class="section">
<h3><a name="Parser_Functions"></a>Parser Functions</h3>
+<div class="section">
+<h4><a name="PARSER_CONFIG"></a><tt>PARSER_CONFIG</tt></h4>
<ul>
-<li><tt>PARSER_STELLAR_TRANSFORM_ADD</tt>
+<li>Description: Returns the configuration of the parser.</li>
+<li>Input:
+<ul>
+
+<li>parser - The parser created with PARSER_INIT.</li>
+</ul>
+</li>
+<li>Returns: The parser configuration.</li>
+</ul>
+<div class="section">
+<h5><a name="Example"></a>Example</h5>
+<ol style="list-style-type: decimal">
+
+<li>
+
+<p>Initialize the parser. The parser configuration for the sensor ‘bro’ will be loaded automatically from Zookeeper.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> parser := PARSER_INIT("bro")
+ Parser{0 successful, 0 error(s)}
+</pre></div></div>
+</li>
+<li>
+
+<p>Review the parser configuration for the ‘bro’ sensor.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> PARSER_CONFIG(parser)
+ {
+ "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
+ "filterClassName":"org.apache.metron.parsers.filters.StellarFilter",
+ "sensorTopic":"bro"
+ }
+</pre></div></div>
+</li>
+</ol></div></div>
+<div class="section">
+<h4><a name="PARSER_INIT"></a><tt>PARSER_INIT</tt></h4>
+<ul>
+
+<li>Initialize a parser to parse the raw telemetry produced by a sensor.</li>
+<li>Input:
+<ul>
+
+<li>sensorType - The type of sensor to parse.</li>
+<li>config - [Optional] The parser configuration. If not provided, the configuration will be retrieved from Zookeeper.</li>
+</ul>
+</li>
+<li>Returns: A parser that can be used to parse sensor telemetry with <tt>PARSER_PARSE</tt>.</li>
+</ul>
+<div class="section">
+<h5><a name="Example"></a>Example</h5>
+<ol style="list-style-type: decimal">
+
+<li>
+
+<p>Initialize a parser. The parser configuration for the sensor ‘bro’ will be loaded automatically from Zookeeper.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> parser := PARSER_INIT("bro")
+ Parser{0 successful, 0 error(s)}
+</pre></div></div>
+</li>
+<li>
+
+<p>The parser will track the number of successes and failures and echo that information.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> parser
+ Parser{0 successful, 0 error(s)}
+</pre></div></div>
+</li>
+<li>
+
+<p>Alternatively, explicitly define the parser configuration. This can be useful when creating a new parser or testing changes to an existing parser.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> config := SHELL_EDIT()
+ {
+ "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
+ "filterClassName":"org.apache.metron.parsers.filters.StellarFilter",
+ "sensorTopic":"bro"
+ }
+</pre></div></div>
+</li>
+<li>
+
+<p>Initialize the parser with the parser configuration.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> parser := PARSER_INIT("bro", config)
+ Parser{0 successful, 0 error(s)}
+</pre></div></div>
+</li>
+</ol></div></div>
+<div class="section">
+<h4><a name="PARSER_PARSE"></a><tt>PARSER_PARSE</tt></h4>
+<ul>
+
+<li>Parses the raw telemetry produced by a sensor.</li>
+<li>Input:
+<ul>
+
+<li>parser - The parser created with PARSER_INIT.</li>
+<li>input - A telemetry message or list of messages to parse.</li>
+</ul>
+</li>
+<li>Returns: A list of messages that result from parsing the input telemetry. If the input cannot be parsed, a message encapsulating the error is returned as part of that list.</li>
+</ul>
+<div class="section">
+<h5><a name="Example"></a>Example</h5>
+<ol style="list-style-type: decimal">
+
+<li>
+
+<p>Initialize the parser. The parser configuration for the sensor ‘bro’ will be loaded automatically from Zookeeper.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> parser := PARSER_INIT("bro")
+ Parser{0 successful, 0 error(s)}
+</pre></div></div>
+</li>
+<li>
+
+<p>Grab the raw telemetry from the ‘bro’ input topic. You could also mock-up a string that you would like to parse using <tt>SHELL_EDIT</tt>.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> to_parse := KAFKA_GET('bro')
+ [{"http": {"ts":1542313125.807068,"uid":"CUrRne3iLIxXavQtci","id.orig_h"...
+</pre></div></div>
+</li>
+<li>
+
+<p>Parse the telemetry.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> msgs := PARSER_PARSE(parser, to_parse)
+ [{"bro_timestamp":"1542313125.807068","method":"GET","ip_dst_port":8080,...
+</pre></div></div>
+</li>
+<li>
+
+<p>The parser will tally the success.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> parser
+ Parser{1 successful, 0 error(s)}
+</pre></div></div>
+</li>
+<li>
+
+<p>Review the successfully parsed message.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> LENGTH(msgs)
+ 1
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> msg := GET(msgs, 0)
+
+ [Stellar]>>> MAP_GET("guid", msg)
+ 7f2e0c77-c58c-488e-b1ad-fbec10fb8182
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> MAP_GET("timestamp", msg)
+ 1542313125807
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> MAP_GET("source.type", msg)
+ bro
+</pre></div></div>
+</li>
+<li>
+
+<p>Attempt to parse invalid input. This will return the error message that is pushed onto the error topic. The error message contains all of the details indicating why parsing failed.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> errors := PARSER_PARSE(parser, "{invalid>")
+</pre></div></div>
+</li>
+<li>
+
+<p>Review the details of the error.</p>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> error := GET(errors, 0)
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> MAP_GET("raw_message", error)
+ {invalid>
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> MAP_GET("message", error)
+ Unable to parse Message: {invalid>
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source"> [Stellar]>>> MAP_GET("stack", error)
+ java.lang.IllegalStateException: Unable to parse Message: {invalid>
+ at org.apache.metron.parsers.bro.BasicBroParser.parse(BasicBroParser.java:145)
+ ...
+</pre></div></div>
+</li>
+</ol></div></div>
+<div class="section">
+<h4><a name="PARSER_STELLAR_TRANSFORM_ADD"></a><tt>PARSER_STELLAR_TRANSFORM_ADD</tt></h4>
<ul>
<li>Description: Add stellar field transformation.</li>
@@ -390,9 +702,9 @@
</ul>
</li>
<li>Returns: The String representation of the config in zookeeper</li>
-</ul>
-</li>
-<li><tt>PARSER-STELLAR_TRANSFORM_PRINT</tt>
+</ul></div>
+<div class="section">
+<h4><a name="PARSER_STELLAR_TRANSFORM_PRINT"></a><tt>PARSER_STELLAR_TRANSFORM_PRINT</tt></h4>
<ul>
<li>Description: Retrieve stellar field transformations.</li>
@@ -403,9 +715,9 @@
</ul>
</li>
<li>Returns: The String representation of the transformations</li>
-</ul>
-</li>
-<li><tt>PARSER_STELLAR_TRANSFORM_REMOVE</tt>
+</ul></div>
+<div class="section">
+<h4><a name="PARSER_STELLAR_TRANSFORM_REMOVE"></a><tt>PARSER_STELLAR_TRANSFORM_REMOVE</tt></h4>
<ul>
<li>Description: Remove stellar field transformation.</li>
@@ -417,14 +729,11 @@
</ul>
</li>
<li>Returns: The String representation of the config in zookeeper</li>
-</ul>
-</li>
-</ul></div>
+</ul></div></div>
<div class="section">
<h3><a name="Indexing_Functions"></a>Indexing Functions</h3>
-<ul>
-
-<li><tt>INDEXING_SET_BATCH</tt>
+<div class="section">
+<h4><a name="INDEXING_SET_BATCH"></a><tt>INDEXING_SET_BATCH</tt></h4>
<ul>
<li>Description: Set batch size and timeout</li>
@@ -438,9 +747,9 @@
</ul>
</li>
<li>Returns: The String representation of the config in zookeeper</li>
-</ul>
-</li>
-<li><tt>INDEXING_SET_ENABLED</tt>
+</ul></div>
+<div class="section">
+<h4><a name="INDEXING_SET_ENABLED"></a><tt>INDEXING_SET_ENABLED</tt></h4>
<ul>
<li>Description: Enable or disable an indexing writer for a sensor.</li>
@@ -453,9 +762,9 @@
</ul>
</li>
<li>Returns: The String representation of the config in zookeeper</li>
-</ul>
-</li>
-<li><tt>INDEXING_SET_INDEX</tt>
+</ul></div>
+<div class="section">
+<h4><a name="INDEXING_SET_INDEX"></a><tt>INDEXING_SET_INDEX</tt></h4>
<ul>
<li>Description: Set the index for the sensor</li>
@@ -468,14 +777,11 @@
</ul>
</li>
<li>Returns: The String representation of the config in zookeeper</li>
-</ul>
-</li>
-</ul></div>
+</ul></div></div>
<div class="section">
<h3><a name="Enrichment_Functions"></a>Enrichment Functions</h3>
-<ul>
-
-<li><tt>ENRICHMENT_STELLAR_TRANSFORM_ADD</tt>
+<div class="section">
+<h4><a name="ENRICHMENT_STELLAR_TRANSFORM_ADD"></a><tt>ENRICHMENT_STELLAR_TRANSFORM_ADD</tt></h4>
<ul>
<li>Description: Add stellar field transformation.</li>
@@ -489,9 +795,9 @@
</ul>
</li>
<li>Returns: The String representation of the config in zookeeper</li>
-</ul>
-</li>
-<li><tt>ENRICHMENT_STELLAR_TRANSFORM_PRINT</tt>
+</ul></div>
+<div class="section">
+<h4><a name="ENRICHMENT_STELLAR_TRANSFORM_PRINT"></a><tt>ENRICHMENT_STELLAR_TRANSFORM_PRINT</tt></h4>
<ul>
<li>Description: Retrieve stellar enrichment transformations.</li>
@@ -503,9 +809,9 @@
</ul>
</li>
<li>Returns: The String representation of the transformations</li>
-</ul>
-</li>
-<li><tt>ENRICHMENT_STELLAR_TRANSFORM_REMOVE</tt>
+</ul></div>
+<div class="section">
+<h4><a name="ENRICHMENT_STELLAR_TRANSFORM_REMOVE"></a><tt>ENRICHMENT_STELLAR_TRANSFORM_REMOVE</tt></h4>
<ul>
<li>Description: Remove one or more stellar field transformations.</li>
@@ -519,14 +825,11 @@
</ul>
</li>
<li>Returns: The String representation of the config in zookeeper</li>
-</ul>
-</li>
-</ul></div>
+</ul></div></div>
<div class="section">
<h3><a name="Threat_Triage_Functions"></a>Threat Triage Functions</h3>
-<ul>
-
-<li><tt>THREAT_TRIAGE_INIT</tt>
+<div class="section">
+<h4><a name="THREAT_TRIAGE_INIT"></a><tt>THREAT_TRIAGE_INIT</tt></h4>
<ul>
<li>Description: Create a threat triage engine.</li>
@@ -537,9 +840,9 @@
</ul>
</li>
<li>Returns: A threat triage engine.</li>
-</ul>
-</li>
-<li><tt>THREAT_TRIAGE_CONFIG</tt>
+</ul></div>
+<div class="section">
+<h4><a name="THREAT_TRIAGE_CONFIG"></a><tt>THREAT_TRIAGE_CONFIG</tt></h4>
<ul>
<li>Description: Export the configuration used by a threat triage engine.</li>
@@ -550,9 +853,9 @@
</ul>
</li>
<li>Returns: The configuration used by the threat triage engine.</li>
-</ul>
-</li>
-<li><tt>THREAT_TRIAGE_SCORE</tt>
+</ul></div>
+<div class="section">
+<h4><a name="THREAT_TRIAGE_SCORE"></a><tt>THREAT_TRIAGE_SCORE</tt></h4>
<ul>
<li>Description: Scores a message using a set of triage rules.</li>
@@ -564,9 +867,9 @@
</ul>
</li>
<li>Returns: A threat triage engine.</li>
-</ul>
-</li>
-<li><tt>THREAT_TRIAGE_ADD</tt>
+</ul></div>
+<div class="section">
+<h4><a name="THREAT_TRIAGE_ADD"></a><tt>THREAT_TRIAGE_ADD</tt></h4>
<ul>
<li>Description: Add a threat triage rule.</li>
@@ -579,9 +882,9 @@
</ul>
</li>
<li>Returns: The String representation of the threat triage rules</li>
-</ul>
-</li>
-<li><tt>THREAT_TRIAGE_REMOVE</tt>
+</ul></div>
+<div class="section">
+<h4><a name="THREAT_TRIAGE_REMOVE"></a><tt>THREAT_TRIAGE_REMOVE</tt></h4>
<ul>
<li>Description: Remove stellar threat triage rule(s).</li>
@@ -593,9 +896,9 @@
</ul>
</li>
<li>Returns: The String representation of the enrichment config</li>
-</ul>
-</li>
-<li><tt>THREAT_TRIAGE_PRINT</tt>
+</ul></div>
+<div class="section">
+<h4><a name="THREAT_TRIAGE_PRINT"></a><tt>THREAT_TRIAGE_PRINT</tt></h4>
<ul>
<li>Description: Retrieve stellar enrichment transformations.</li>
@@ -606,9 +909,9 @@
</ul>
</li>
<li>Returns: The String representation of the threat triage rules</li>
-</ul>
-</li>
-<li><tt>THREAT_TRIAGE_SET_AGGREGATOR</tt>
+</ul></div>
+<div class="section">
+<h4><a name="THREAT_TRIAGE_SET_AGGREGATOR"></a><tt>THREAT_TRIAGE_SET_AGGREGATOR</tt></h4>
<ul>
<li>Description: Set the threat triage aggregator.</li>
@@ -622,8 +925,138 @@
</li>
<li>Returns: The String representation of the enrichment config</li>
</ul>
+<div class="section">
+<h5><a name="Example"></a>Example</h5>
+<ol style="list-style-type: decimal">
+
+<li>
+
+<p>Create a threat triage engine.</p>
+
+<div>
+<div>
+<pre class="source">[Stellar]>>> t := THREAT_TRIAGE_INIT()
+[Stellar]>>> t
+ThreatTriage{0 rule(s)}
+</pre></div></div>
</li>
-</ul></div></div>
+<li>
+
+<p>Add a few triage rules.</p>
+
+<div>
+<div>
+<pre class="source">[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule1", "rule":"value>10", "score":10})
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source">[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule2", "rule":"value>20", "score":20})
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source">[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule3", "rule":"value>30", "score":30})
+</pre></div></div>
+</li>
+<li>
+
+<p>Review the rules that you have created.</p>
+
+<div>
+<div>
+<pre class="source">[Stellar]>>> THREAT_TRIAGE_PRINT(t)
+╔═══════╤═════════╤═════════════╤═══════╤════════╗
+║ Name │ Comment │ Triage Rule │ Score │ Reason ║
+╠═══════╪═════════╪═════════════╪═══════╪════════╣
+║ rule1 │ │ value>10 │ 10 │ ║
+╟───────┼─────────┼─────────────┼───────┼────────╢
+║ rule2 │ │ value>20 │ 20 │ ║
+╟───────┼─────────┼─────────────┼───────┼────────╢
+║ rule3 │ │ value>30 │ 30 │ ║
+╚═══════╧═════════╧═════════════╧═══════╧════════╝
+</pre></div></div>
+</li>
+<li>
+
+<p>Create a few test messages to simulate your telemetry.</p>
+
+<div>
+<div>
+<pre class="source">[Stellar]>>> msg1 := "{ \"value\":22 }"
+[Stellar]>>> msg1
+{ "value":22 }
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source">[Stellar]>>> msg2 := "{ \"value\":44 }"
+[Stellar]>>> msg2
+{ "value":44 }
+</pre></div></div>
+</li>
+<li>
+
+<p>Score a message based on the rules that have been defined. The result allows you to see the total score, the aggregator, along with details about each rule that fired.</p>
+
+<div>
+<div>
+<pre class="source">[Stellar]>>> THREAT_TRIAGE_SCORE( msg1, t)
+{score=20.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value>10}, {score=20.0, name=rule2, rule=value>20}]}
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source">[Stellar]>>> THREAT_TRIAGE_SCORE( msg2, t)
+{score=30.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value>10}, {score=20.0, name=rule2, rule=value>20}, {score=30.0, name=rule3, rule=value>30}]}
+</pre></div></div>
+</li>
+<li>
+
+<p>From here you can iterate on your rule set until it does exactly what you need it to do. Once you have a working rule set, extract the configuration and push it into your live, Metron cluster.</p>
+
+<div>
+<div>
+<pre class="source">[Stellar]>>> conf := THREAT_TRIAGE_CONFIG( t)
+[Stellar]>>> conf
+{
+ "enrichment" : {
+ "fieldMap" : { },
+ "fieldToTypeMap" : { },
+ "config" : { }
+ },
+ "threatIntel" : {
+ "fieldMap" : { },
+ "fieldToTypeMap" : { },
+ "config" : { },
+ "triageConfig" : {
+ "riskLevelRules" : [ {
+ "name" : "rule1",
+ "rule" : "value>10",
+ "score" : 10.0
+ }, {
+ "name" : "rule2",
+ "rule" : "value>20",
+ "score" : 20.0
+ }, {
+ "name" : "rule3",
+ "rule" : "value>30",
+ "score" : 30.0
+ }],
+ "aggregator" : "MAX",
+ "aggregationConfig" : { }
+ }
+ },
+ "configuration" : { }
+}
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source">[Stellar]>>> CONFIG_PUT("ENRICHMENT", conf, "bro")
+</pre></div></div>
+</li>
+</ol></div></div></div></div>
<div class="section">
<h2><a name="Deployment_Instructions"></a>Deployment Instructions</h2>
<ul>
@@ -652,7 +1085,7 @@
<pre class="source">Stellar, Go!
Please note that functions are loading lazily in the background and will be unavailable until loaded fully.
[Stellar]>>> # We are going to debug a squid grok statement with a bug in it
-[Stellar]>>> squid_grok_orig := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}
+[Stellar]>>> squid_grok_orig := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}
- %{WORD:UNWANTED}/%{IP:ip_dst_addr} %{WORD:UNWANTED}/%{WORD:UNWANTED}'
[Stellar]>>> # We have gone ot a couple of domains in squid:
[Stellar]>>> # 1475022887.362 256 127.0.0.1 TCP_MISS/301 803 GET http://www.youtube.com/ - DIRECT/216.58.216.238 text/html
@@ -661,7 +1094,7 @@
[Stellar]>>> # Note that flimflammadeupdomain.com and www.google.com did not resolve to IPs
[Stellar]>>> # We can load up these messages from disk into a list of messages
[Stellar]>>> messages := LOCAL_READ_LINES( '/var/log/squid/access.log')
-27687 [Thread-1] INFO o.r.Reflections - Reflections took 26542 ms to scan 22 urls, producing 17906 keys and 121560 values
+27687 [Thread-1] INFO o.r.Reflections - Reflections took 26542 ms to scan 22 urls, producing 17906 keys and 121560 values
27837 [Thread-1] INFO o.a.m.c.d.FunctionResolverSingleton - Found 97 Stellar Functions...
Functions loaded, you may refer to functions now...
[Stellar]>>> # and evaluate the messages against our grok statement
@@ -678,7 +1111,7 @@
[Stellar]>>> # Uh oh, looks like the messages without destination IPs do not parse
[Stellar]>>> # We can start peeling off groups from the end of the message until things parse
-[Stellar]>>> squid_grok := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{
+[Stellar]>>> squid_grok := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{
WORD:UNWANTED}/%{IP:ip_dst_addr}'
[Stellar]>>> GROK_EVAL(squid_grok, messages)
╔══════════╤═════════╤═════════╤═════════╤════════════════╤═════════════╤═════════╤════════════════╤═════════════════════════╗
@@ -692,7 +1125,7 @@
╚══════════╧═════════╧═════════╧═════════╧════════════════╧═════════════╧═════════╧════════════════╧═════════════════════════╝
[Stellar]>>> # Still looks like it is having issues...
-[Stellar]>>> squid_grok := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{
+[Stellar]>>> squid_grok := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{
WORD:UNWANTED}/%{IP:ip_dst_addr}'
[Stellar]>>> GROK_EVAL(squid_grok, messages)
╔══════════╤═════════╤═════════╤═════════╤════════════════╤═════════════╤═════════╤════════════════╤═════════════════════════╗
@@ -706,7 +1139,7 @@
╚══════════╧═════════╧═════════╧═════════╧════════════════╧═════════════╧═════════╧════════════════╧═════════════════════════╝
[Stellar]>>> # Still looks wrong. Hmm, I bet it is due to that dst_addr not being there; we can make it optional
-[Stellar]>>> squid_grok := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{
+[Stellar]>>> squid_grok := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{
WORD:UNWANTED}/(%{IP:ip_dst_addr})?'
[Stellar]>>> GROK_EVAL(squid_grok, messages)
╔══════════╤═══════╤══════╤═════════╤════════════════╤═════════════╤════════╤════════════════╤══════════════════════════╗
@@ -720,8 +1153,8 @@
╚══════════╧═══════╧══════╧═════════╧════════════════╧═════════════╧════════╧════════════════╧══════════════════════════╝
[Stellar]>>> # Ahh, that is much better.
-[Stellar]>>>
-[Stellar]>>>
+[Stellar]>>>
+[Stellar]>>>
</pre></div></div>
</div>
<div class="section">
@@ -736,7 +1169,7 @@
{es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH}
[Stellar]>>> # First we get the squid parser config from zookeeper
[Stellar]>>> squid_parser_config := CONFIG_GET('PARSER', 'squid')
-29089 [Thread-1] INFO o.r.Reflections - Reflections took 26765 ms to scan 22 urls, producing 17898 keys and 121518 values
+29089 [Thread-1] INFO o.r.Reflections - Reflections took 26765 ms to scan 22 urls, producing 17898 keys and 121518 values
29177 [Thread-1] INFO o.a.m.c.d.FunctionResolverSingleton - Found 83 Stellar Functions...
Functions loaded, you may refer to functions now...
[Stellar]>>> # See what kind of transformations it already has
@@ -774,7 +1207,7 @@
[Stellar]>>> # Add another transformation in there
[Stellar]>>> domain_without_subdomains := 'cnn.com'
[Stellar]>>> upper_domain := TO_UPPER(domain_without_subdomains)
-[Stellar]>>> # Now we can look at our variables and see what expressions created them
+[Stellar]>>> # Now we can look at our variables and see what expressions created them
[Stellar]>>> # NOTE: the 40 is the max char for a word
[Stellar]>>> SHELL_LIST_VARS( 40 )
╔═══════════════════════════╤═══════════════════════════════════════════╤═════════════════════════════════════╗
@@ -926,7 +1359,7 @@
[Stellar]>>> # If it is not there, which it is not by default, a suitable default
[Stellar]>>> # config will be specified.
[Stellar]>>> squid_enrichment_config := CONFIG_GET('ENRICHMENT', 'squid')
-26307 [Thread-1] INFO o.r.Reflections - Reflections took 24845 ms to scan 22 urls, producing 17898 keys and 121520 values
+26307 [Thread-1] INFO o.r.Reflections - Reflections took 24845 ms to scan 22 urls, producing 17898 keys and 121520 values
26389 [Thread-1] INFO o.a.m.c.d.FunctionResolverSingleton - Found 84 Stellar Functions...
Functions loaded, you may refer to functions now...
[Stellar]>>> # Just to make sure it looks right, we can view the JSON
@@ -1107,7 +1540,7 @@
Returns: The String representation of the config in zookeeper
[Stellar]>>> squid_enrichment_config_new := ENRICHMENT_STELLAR_TRANSFORM_REMOVE( squid_enrichment_config_new, 'ENRICHMENT', [ 'is_local' ] )
-[Stellar]>>> # Make sure that it is really gone
+[Stellar]>>> # Make sure that it is really gone
[Stellar]>>> ENRICHMENT_STELLAR_TRANSFORM_PRINT(squid_enrichment_config_new, 'ENRICHMENT')
╔═══════╤═══════╤════════════════╗
║ Group │ Field │ Transformation ║
@@ -1146,7 +1579,7 @@
},
"configuration" : { }
}
-[Stellar]>>>
+[Stellar]>>>
</pre></div></div>
</div>
<div class="section">
@@ -1163,7 +1596,7 @@
[Stellar]>>> # If it is not there, which it is not by default, a suitable default
[Stellar]>>> # config will be specified.
[Stellar]>>> squid_enrichment_config := CONFIG_GET('ENRICHMENT', 'squid')
-26751 [Thread-1] INFO o.r.Reflections - Reflections took 24407 ms to scan 22 urls, producing 17898 keys and 121520 values
+26751 [Thread-1] INFO o.r.Reflections - Reflections took 24407 ms to scan 22 urls, producing 17898 keys and 121520 values
26828 [Thread-1] INFO o.a.m.c.d.FunctionResolverSingleton - Found 84 Stellar Functions...
Functions loaded, you may refer to functions now...
[Stellar]>>> # We should not have any threat triage rules
@@ -1175,7 +1608,7 @@
╚═════════════════════╝
[Stellar]>>> # I have followed the blog post at https://cwiki.apache.org/confluence/display/METRON/2016/04/28/Metron+Tutorial+-+Fundamentals+Part+2%3A+Creating+a+New+Enrichment
-[Stellar]>>> # and have some enrichment reference data loaded into hbase,
+[Stellar]>>> # and have some enrichment reference data loaded into hbase,
[Stellar]>>> # so we should be able to retrieve that as an enrichment using the ENRICHMENT_GET
[Stellar]>>> # function to call out to hbase from stellar
[Stellar]>>> ?ENRICHMENT_GET
@@ -1311,7 +1744,7 @@
"configuration" : { }
}
[Stellar]>>> # Now that we have admired it, we can remove the rules
-[Stellar]>>> squid_enrichment_config_new := THREAT_TRIAGE_REMOVE( squid_enrichment_config_new, [ SHELL_GET_EXPRESSION('non_us') , SHELL_GET_EXPRESSION('is_local') , SHELL_GET_EXPRES
+[Stellar]>>> squid_enrichment_config_new := THREAT_TRIAGE_REMOVE( squid_enrichment_config_new, [ SHELL_GET_EXPRESSION('non_us') , SHELL_GET_EXPRESSION('is_local') , SHELL_GET_EXPRES
SION('is_both') ] )
[Stellar]>>> THREAT_TRIAGE_PRINT(squid_enrichment_config_new)
╔══════╤═════════╤═════════════╤═══════╗
@@ -1354,141 +1787,8 @@
},
"configuration" : { }
}
-[Stellar]>>>
-</pre></div></div>
-</div>
-<div class="section">
-<h3><a name="Simulate_Threat_Triage_Rules"></a>Simulate Threat Triage Rules</h3>
-<ol style="list-style-type: decimal">
-
-<li>
-
-<p>Create a threat triage engine.</p>
-
-<div>
-<div>
-<pre class="source">[Stellar]>>> t := THREAT_TRIAGE_INIT()
-[Stellar]>>> t
-ThreatTriage{0 rule(s)}
-</pre></div></div>
-</li>
-<li>
-
-<p>Add a few triage rules.</p>
-
-<div>
-<div>
-<pre class="source">[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule1", "rule":"value>10", "score":10})
-</pre></div></div>
-
-<div>
-<div>
-<pre class="source">[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule2", "rule":"value>20", "score":20})
-</pre></div></div>
-
-<div>
-<div>
-<pre class="source">[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule3", "rule":"value>30", "score":30})
-</pre></div></div>
-</li>
-<li>
-
-<p>Review the rules that you have created.</p>
-
-<div>
-<div>
-<pre class="source">[Stellar]>>> THREAT_TRIAGE_PRINT(t)
-╔═══════╤═════════╤═════════════╤═══════╤════════╗
-║ Name │ Comment │ Triage Rule │ Score │ Reason ║
-╠═══════╪═════════╪═════════════╪═══════╪════════╣
-║ rule1 │ │ value>10 │ 10 │ ║
-╟───────┼─────────┼─────────────┼───────┼────────╢
-║ rule2 │ │ value>20 │ 20 │ ║
-╟───────┼─────────┼─────────────┼───────┼────────╢
-║ rule3 │ │ value>30 │ 30 │ ║
-╚═══════╧═════════╧═════════════╧═══════╧════════╝
-</pre></div></div>
-</li>
-<li>
-
-<p>Create a few test messages to simulate your telemetry.</p>
-
-<div>
-<div>
-<pre class="source">[Stellar]>>> msg1 := "{ \"value\":22 }"
-[Stellar]>>> msg1
-{ "value":22 }
-</pre></div></div>
-
-<div>
-<div>
-<pre class="source">[Stellar]>>> msg2 := "{ \"value\":44 }"
-[Stellar]>>> msg2
-{ "value":44 }
-</pre></div></div>
-</li>
-<li>
-
-<p>Score a message based on the rules that have been defined. The result allows you to see the total score, the aggregator, along with details about each rule that fired.</p>
-
-<div>
-<div>
-<pre class="source">[Stellar]>>> THREAT_TRIAGE_SCORE( msg1, t)
-{score=20.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value>10}, {score=20.0, name=rule2, rule=value>20}]}
-</pre></div></div>
-
-<div>
-<div>
-<pre class="source">[Stellar]>>> THREAT_TRIAGE_SCORE( msg2, t)
-{score=30.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value>10}, {score=20.0, name=rule2, rule=value>20}, {score=30.0, name=rule3, rule=value>30}]}
-</pre></div></div>
-</li>
-<li>
-
-<p>From here you can iterate on your rule set until it does exactly what you need it to do. Once you have a working rule set, extract the configuration and push it into your live, Metron cluster.</p>
-
-<div>
-<div>
-<pre class="source">[Stellar]>>> conf := THREAT_TRIAGE_CONFIG( t)
-[Stellar]>>> conf
-{
- "enrichment" : {
- "fieldMap" : { },
- "fieldToTypeMap" : { },
- "config" : { }
- },
- "threatIntel" : {
- "fieldMap" : { },
- "fieldToTypeMap" : { },
- "config" : { },
- "triageConfig" : {
- "riskLevelRules" : [ {
- "name" : "rule1",
- "rule" : "value>10",
- "score" : 10.0
- }, {
- "name" : "rule2",
- "rule" : "value>20",
- "score" : 20.0
- }, {
- "name" : "rule3",
- "rule" : "value>30",
- "score" : 30.0
- }],
- "aggregator" : "MAX",
- "aggregationConfig" : { }
- }
- },
- "configuration" : { }
-}
-</pre></div></div>
-
-<div>
-<div>
-<pre class="source">[Stellar]>>> CONFIG_PUT("ENRICHMENT", conf, "bro")
-</pre></div></div>
-</li>
-</ol></div></div>
+[Stellar]>>>
+</pre></div></div></div></div>
</div>
</div>
</div>
diff --git a/current-book/metron-platform/metron-parsing/index.html b/current-book/metron-platform/metron-parsing/index.html
new file mode 100644
index 0000000..3f96fbf
--- /dev/null
+++ b/current-book/metron-platform/metron-parsing/index.html
@@ -0,0 +1,873 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsing/index.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Parsers</title>
+ <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../css/site.css" />
+ <link rel="stylesheet" href="../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">Parsers</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li class="active"><a href="#"><span class="icon-chevron-down"></span>Parsing</a>
+ <ul class="nav nav-list">
+ <li><a href="../../metron-platform/metron-parsing/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../metron-platform/metron-parsing/metron-parsers-common/index.html" title="Parsers-common"><span class="icon-chevron-right"></span>Parsers-common</a></li>
+ <li><a href="../../metron-platform/metron-parsing/metron-parsing-storm/index.html" title="Parsing-storm"><span class="none"></span>Parsing-storm</a></li>
+ </ul>
+</li>
+ <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Parsers</h1>
+<p><a name="Parsers"></a></p>
+<div class="section">
+<h2><a name="Contents"></a>Contents</h2>
+<ul>
+
+<li><a href="#Introduction">Introduction</a></li>
+<li><a href="#Parser_Error_Routing">Parser Error Routing</a></li>
+<li><a href="#Filtering">Filtering</a></li>
+<li><a href="#Parser_Architecture">Parser Architecture</a></li>
+<li><a href="#Message_Format">Message Format</a></li>
+<li><a href="#Global_Configuration">Global Configuration</a></li>
+<li><a href="#Parser_Configuration">Parser Configuration</a></li>
+<li><a href="#Parser_Adapters">Parser Adapters</a></li>
+<li><a href="#Kafka_Queue">Kafka Queue</a></li>
+<li><a href="#JSON_Path">JSON Path</a></li>
+</ul></div>
+<div class="section">
+<h2><a name="Introduction"></a>Introduction</h2>
+<p>Parsers are pluggable components which are used to transform raw data (textual or raw bytes) into JSON messages suitable for downstream enrichment and indexing.</p>
+<p>There are two general types types of parsers:</p>
+<ul>
+
+<li>A parser written in Java which conforms to the <tt>MessageParser</tt> interface. This kind of parser is optimized for speed and performance and is built for use with higher velocity topologies. These parsers are not easily modifiable and in order to make changes to them the entire topology need to be recompiled.</li>
+<li>A general purpose parser. This type of parser is primarily designed for lower-velocity topologies or for quickly standing up a parser for a new telemetry before a permanent Java parser can be written for it. As of the time of this writing, we have:
+<ul>
+
+<li>Grok parser: <tt>org.apache.metron.parsers.GrokParser</tt> with possible <tt>parserConfig</tt> entries of
+<ul>
+
+<li><tt>grokPath</tt> : The path in HDFS (or in the Jar) to the grok statement. By default attempts to load from HDFS, then falls back to the classpath, and finally throws an exception if unable to load a pattern.</li>
+<li><tt>patternLabel</tt> : The pattern label to use from the grok statement</li>
+<li><tt>multiLine</tt> : The raw data passed in should be handled as a long with multiple lines, with each line to be parsed separately. This setting’s valid values are ‘true’ or ‘false’. The default if unset is ‘false’. When set the parser will handle multiple lines with successfully processed lines emitted normally, and lines with errors sent to the error topic.</li>
+<li><tt>timestampField</tt> : The field to use for timestamp. If your data does not have a field exactly named “timestamp” this field is required, otherwise the record will not pass validation. If the timestampField is also included in the list of timeFields, it will first be parsed using the provided dateFormat.</li>
+<li><tt>timeFields</tt> : A list of fields to be treated as time.</li>
+<li><tt>dateFormat</tt> : The date format to use to parse the time fields. Default is “yyyy-MM-dd HH:mm:ss.S z”.</li>
+<li><tt>timezone</tt> : The timezone to use. <tt>UTC</tt> is default.</li>
+<li>The Grok parser supports either 1 line to parse per incoming message, or incoming messages with multiple log lines, and will produce a json message per line</li>
+</ul>
+</li>
+<li>CSV Parser: <tt>org.apache.metron.parsers.csv.CSVParser</tt> with possible <tt>parserConfig</tt> entries of
+<ul>
+
+<li><tt>timestampFormat</tt> : The date format of the timestamp to use. If unspecified, the parser assumes the timestamp is ms since unix epoch.</li>
+<li><tt>columns</tt> : A map of column names you wish to extract from the CSV to their offsets (e.g. <tt>{ 'name' : 1, 'profession' : 3}</tt> would be a column map for extracting the 2nd and 4th columns from a CSV)</li>
+<li><tt>separator</tt> : The column separator, <tt>,</tt> by default.</li>
+</ul>
+</li>
+<li>JSON Map Parser: <tt>org.apache.metron.parsers.json.JSONMapParser</tt> with possible <tt>parserConfig</tt> entries of
+<ul>
+
+<li><tt>mapStrategy</tt> : A strategy to indicate how to handle multi-dimensional Maps. This is one of
+<ul>
+
+<li><tt>DROP</tt> : Drop fields which contain maps</li>
+<li><tt>UNFOLD</tt> : Unfold inner maps. So <tt>{ "foo" : { "bar" : 1} }</tt> would turn into <tt>{"foo.bar" : 1}</tt></li>
+<li><tt>ALLOW</tt> : Allow multidimensional maps</li>
+<li><tt>ERROR</tt> : Throw an error when a multidimensional map is encountered</li>
+</ul>
+</li>
+<li><tt>jsonpQuery</tt> : A <a href="#json_path">JSON Path</a> query string. If present, the result of the JSON Path query should be a list of messages. This is useful if you have a JSON document which contains a list or array of messages embedded in it, and you do not have another means of splitting the message.</li>
+<li><tt>wrapInEntityArray</tt> : <tt>"true" or "false"</tt>. If <tt>jsonQuery</tt> is present and this flag is present and set to <tt>"true"</tt>, the incoming message will be wrapped in a JSON entity and array. for example: <tt>{"name":"value"},{"name2","value2}</tt> will be wrapped as <tt>{"message" : [{"name":"value"},{"name2","value2}]}</tt>. This is using the default value for <tt>wrapEntityName</tt> if that property is not set.</li>
+<li><tt>wrapEntityName</tt> : Sets the name to use when wrapping JSON using <tt>wrapInEntityArray</tt>. The <tt>jsonpQuery</tt> should reference this name.</li>
+<li>A field called <tt>timestamp</tt> is expected to exist and, if it does not, then current time is inserted.</li>
+</ul>
+</li>
+<li>Regular Expressions Parser
+<ul>
+
+<li><tt>recordTypeRegex</tt> : A regular expression to uniquely identify a record type.</li>
+<li><tt>messageHeaderRegex</tt> : A regular expression used to extract fields from a message part which is common across all the messages.</li>
+<li><tt>convertCamelCaseToUnderScore</tt> : If this property is set to true, this parser will automatically convert all the camel case property names to underscore seperated. For example, following conversions will automatically happen:
+
+<div>
+<div>
+<pre class="source">ipSrcAddr -> ip_src_addr
+ipDstAddr -> ip_dst_addr
+ipSrcPort -> ip_src_port
+</pre></div></div>
+
+<p>Note this property may be necessary, because java does not support underscores in the named group names. So in case your property naming conventions requires underscores in property names, use this property.</p></li>
+<li>
+
+<p><tt>fields</tt> : A json list of maps contaning a record type to regular expression mapping.</p>
+</li>
+</ul>
+<p>A complete configuration example would look like:</p>
+
+<div>
+<div>
+<pre class="source">"convertCamelCaseToUnderScore": true,
+"recordTypeRegex": "kernel|syslog",
+"messageHeaderRegex": "(<syslogPriority>(<=^&lt;)\\d{1,4}(?=>)).*?(<timestamp>(<=>)[A-Za-z] {3}\\s{1,2}\\d{1,2}\\s\\d{1,2}:\\d{1,2}:\\d{1,2}(?=\\s)).*?(<syslogHost>(<=\\s).*?(?=\\s))",
+"fields": [
+ {
+ "recordType": "kernel",
+ "regex": ".*(<eventInfo>(<=\\]|\\w\\:).*?(?=$))"
+ },
+ {
+ "recordType": "syslog",
+ "regex": ".*(<processid>(<=PID\\s=\\s).*?(?=\\sLine)).*(<filePath>(<=64\\s)\/([A-Za-z0-9_-]+\/)+(?=\\w)) (<fileName>.*?(?=\")).*(<eventInfo>(<=\").*?(?=$))"
+ }
+]
+</pre></div></div>
+
+<p><b>Note</b>: messageHeaderRegex and regex (withing fields) could be specified as lists also e.g.</p>
+
+<div>
+<div>
+<pre class="source">"messageHeaderRegex": [
+ "regular expression 1",
+ "regular expression 2"
+]
+</pre></div></div>
+
+<p>Where <b>regular expression 1</b> are valid regular expressions and may have named groups, which would be extracted into fields. This list will be evaluated in order until a matching regular expression is found.</p>
+<p><b>messageHeaderRegex</b> is run on all the messages. Yes, all the messages are expected to contain the fields which are being extracted using the <b>messageHeaderRegex</b>. <b>messageHeaderRegex</b> is a sort of HCF (highest common factor) in all messages.</p>
+<p><b>recordTypeRegex</b> can be a more advanced regular expression containing named goups. For example</p>
+<p>“recordTypeRegex”: “(<process>(<=\s)\b(kernel|syslog)\b(?=\[|:))”</p>
+<p>Here all the named goups (process in above example) will be extracted as fields.</p>
+<p>Though having named group in recordType is completely optional, still one could want extract named groups in recordType for following reasons:</p>
+<ol style="list-style-type: decimal">
+
+<li>Since <b>recordType</b> regular expression is already getting matched and we are paying the price for a regular expression match already, we can extract certain fields as a by product of this match.</li>
+<li>Most likely the <b>recordType</b> field is common across all the messages. Hence having it extracted in the recordType (or messageHeaderRegex) would reduce the overall complexity of regular expressions in the regex field.</li>
+</ol>
+<p><b>regex</b> within a field could be a list of regular expressions also. In this case all regular expressions in the list will be attempted to match until a match is found. Once a full match is found remaining regular expressions are ignored.</p>
+
+<div>
+<div>
+<pre class="source">"regex": [ "record type specific regular expression 1",
+ "record type specific regular expression 2"]
+</pre></div></div>
+
+<p><b>timesamp</b></p>
+<p>Since this parser is a general purpose parser, it will populate the timestamp field with current UTC timestamp. Actual timestamp value can be overridden later using stellar. For example in case of syslog timestamps, one could use following stellar construct to override the timestamp value. Let us say you parsed actual timestamp from the raw log:</p>
+<p><tt><38>Jun 20 15:01:17 hostName sshd[11672]: Accepted publickey for prod from 55.55.55.55 port 66666 ssh2</tt></p>
+<p>syslogTimestamp=“Jun 20 15:01:17”</p>
+<p>Then something like below could be used to override the timestamp.</p>
+
+<div>
+<div>
+<pre class="source">"timestamp_str": "FORMAT('%s%s%s', YEAR(),' ',syslogTimestamp)",
+"timestamp":"TO_EPOCH_TIMESTAMP(timestamp_str, 'yyyy MMM dd HH:mm:ss' )"
+</pre></div></div>
+
+<p>OR, if you want to factor in the timezone</p>
+
+<div>
+<div>
+<pre class="source">"timestamp":"TO_EPOCH_TIMESTAMP(timestamp_str, timestamp_format, timezone_name )"
+</pre></div></div>
+</li>
+</ul>
+</li>
+</ul></div>
+<div class="section">
+<h2><a name="Parser_Message_Routing"></a>Parser Message Routing</h2>
+<p>Messages are routed to the Kafka <tt>enrichments</tt> topic by default. The output topic can be changed with the <tt>output_topic</tt> option when <a href="metron-parsing-storm/index.html#Starting_the_Parser_Topology">Starting the Parser Topology</a> or with the <tt>outputTopic</tt> <a href="#Parser_Configuration">Parser Configuration</a> setting. The order of precedence from highest to lowest is as follows:</p>
+<ol style="list-style-type: decimal">
+
+<li>Parser start script option</li>
+<li>Parser configuration setting</li>
+<li>Default <tt>enrichments</tt> topic</li>
+</ol>
+<p>A message can also be routed to other locations besides Kafka with the <tt>writerClassName</tt> <a href="#Parser_Configuration">Parser Configuration</a> setting. Messages can be routed independently for each sensor type when configured with <a href="#Parser_Configuration">Parser Configuration</a> settings.</p></div>
+<div class="section">
+<h2><a name="Parser_Error_Routing"></a>Parser Error Routing</h2>
+<p>Currently, we have a few mechanisms for either deferring processing of messages or marking messages as invalid.</p>
+<div class="section">
+<h3><a name="Invalidation_Errors"></a>Invalidation Errors</h3>
+<p>There are two reasons a message will be marked as invalid:</p>
+<ul>
+
+<li>Fail <a href="../../metron-common/index.html#validation-framework">global validation</a></li>
+<li>Fail the parser’s validate function. Generally, that means not having a <tt>timestamp</tt> field or an <tt>original_string</tt> field.</li>
+</ul>
+<p>Those messages which are marked as invalid are sent to the error queue with an indication that they are invalid in the error message. The messages will contain “error_type”:“parser_invalid”. Note, you will not see additional exceptions in the logs for this type of failure, rather the error messages are written directly to the configured error topic. See <a href="../../metron-common/index.html#Topology_Errors">Topology Errors</a> for more.</p></div>
+<div class="section">
+<h3><a name="Parser_Errors"></a>Parser Errors</h3>
+<p>Errors, which are defined as unexpected exceptions happening during the parse, are sent along to the error queue with a message indicating that there was an error in parse along with a stacktrace. This is to distinguish from the invalid messages.</p></div></div>
+<div class="section">
+<h2><a name="Filtering"></a>Filtering</h2>
+<p>One can also filter a message by specifying a <tt>filterClassName</tt> in the parser config. Filtered messages are just dropped rather than passed through.</p></div>
+<div class="section">
+<h2><a name="Parser_Architecture"></a>Parser Architecture</h2>
+<p><img src="../../images/parser_arch.png" alt="Architecture" /></p>
+<p>Data flows through the parser via kafka and into the <tt>enrichments</tt> topology in kafka. Errors are collected with the context of the error (e.g. stacktrace) and original message causing the error and sent to an <tt>error</tt> queue. Invalid messages as determined by global validation functions are also treated as errors and sent to an <tt>error</tt> queue.</p></div>
+<div class="section">
+<h2><a name="Message_Format"></a>Message Format</h2>
+<p>All Metron messages follow a specific format in order to ingest a message. If a message does not conform to this format it will be dropped and put onto an error queue for further examination. The message must be of a JSON format and must have a JSON tag message like so:</p>
+
+<div>
+<div>
+<pre class="source">{"message" : message content}
+</pre></div></div>
+
+<p>Where appropriate there is also a standardization around the 5-tuple JSON fields. This is done so the topology correlation engine further down stream can correlate messages from different topologies by these fields. We are currently working on expanding the message standardization beyond these fields, but this feature is not yet availabe. The standard field names are as follows:</p>
+<ul>
+
+<li>ip_src_addr: layer 3 source IP</li>
+<li>ip_dst_addr: layer 3 dest IP</li>
+<li>ip_src_port: layer 4 source port</li>
+<li>ip_dst_port: layer 4 dest port</li>
+<li>protocol: layer 4 protocol</li>
+<li>timestamp (epoch)</li>
+<li>original_string: A human friendly string representation of the message</li>
+</ul>
+<p>The timestamp and original_string fields are mandatory. The remaining standard fields are optional. If any of the optional fields are not applicable then the field should be left out of the JSON.</p>
+<p>So putting it all together a typical Metron message with all 5-tuple fields present would look like the following:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "message": {
+ "ip_src_addr": xxxx,
+ "ip_dst_addr": xxxx,
+ "ip_src_port": xxxx,
+ "ip_dst_port": xxxx,
+ "protocol": xxxx,
+ "original_string": xxx,
+ "additional-field 1": xxx
+ }
+}
+</pre></div></div>
+</div>
+<div class="section">
+<h2><a name="Global_Configuration"></a>Global Configuration</h2>
+<p>There are a few properties which can be managed in the global configuration that have pertinence to parsers and parsing in general.</p>
+<div class="section">
+<h3><a name="parser.error.topic"></a><tt>parser.error.topic</tt></h3>
+<p>The topic where messages which were unable to be parsed due to error are sent. Error messages will be indexed under a sensor type of <tt>error</tt> and the messages will have the following fields:</p>
+<ul>
+
+<li><tt>sensor.type</tt>: <tt>error</tt></li>
+<li><tt>failed_sensor_type</tt> : The sensor type of the message which wasn’t able to be parsed</li>
+<li><tt>error_type</tt> : The error type, in this case <tt>parser</tt>.</li>
+<li><tt>stack</tt> : The stack trace of the error</li>
+<li><tt>hostname</tt> : The hostname of the node where the error happened</li>
+<li><tt>raw_message</tt> : The raw message in string form</li>
+<li><tt>raw_message_bytes</tt> : The raw message bytes</li>
+<li><tt>error_hash</tt> : A hash of the error message</li>
+</ul>
+<p>When aggregating multiple sensors, all sensors must be using the same error topic.</p></div></div>
+<div class="section">
+<h2><a name="Parser_Configuration"></a>Parser Configuration</h2>
+<p>The configuration for the various parser topologies is defined by JSON documents stored in zookeeper.</p>
+<p>The document is structured in the following way</p>
+<ul>
+
+<li><tt>parserClassName</tt> : The fully qualified classname for the parser to be used.</li>
+<li><tt>filterClassName</tt> : The filter to use. This may be a fully qualified classname of a Class that implements the <tt>org.apache.metron.parsers.interfaces.MessageFilter<JSONObject></tt> interface. Message Filters are intended to allow the user to ignore a set of messages via custom logic. The existing implementations are:
+<ul>
+
+<li><tt>STELLAR</tt> : Allows you to apply a stellar statement which returns a boolean, which will pass every message for which the statement returns <tt>true</tt>. The Stellar statement that is to be applied is specified by the <tt>filter.query</tt> property in the <tt>parserConfig</tt>.
+<p>Example Stellar Filter which includes messages which contain a the <tt>field1</tt> field:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "filterClassName" : "STELLAR",
+ "parserConfig" : {
+ "filter.query" : "exists(field1)"
+ }
+}
+</pre></div></div>
+</li>
+</ul>
+</li>
+<li>
+
+<p><tt>writerClassName</tt> : The class used to write messages after they have been parsed. Defaults to <tt>org.apache.metron.writer.kafka.KafkaWriter</tt>.</p>
+</li>
+<li><tt>sensorTopic</tt> : The kafka topic to that the parser will read messages from. If the topic is prefixed and suffixed by <tt>/</tt> then it is assumed to be a regex and will match any topic matching the pattern (e.g. <tt>/bro.*/</tt> would match <tt>bro_cust0</tt>, <tt>bro_cust1</tt> and <tt>bro_cust2</tt>)</li>
+<li><tt>readMetadata</tt> : Boolean indicating whether to read metadata or not (The default is raw message strategy dependent). See below for a discussion about metadata.</li>
+<li><tt>mergeMetadata</tt> : Boolean indicating whether to merge metadata with the message or not (The default is raw message strategy dependent). See below for a discussion about metadata.</li>
+<li><tt>rawMessageStrategy</tt> : The strategy to use when reading the raw data and metadata. See below for a discussion about message reading strategies.</li>
+<li><tt>rawMessageStrategyConfig</tt> : The raw message strategy configuration map. See below for a discussion about message reading strategies.</li>
+<li><tt>parserConfig</tt> : A JSON Map representing the parser implementation specific configuration. Also include batch sizing and timeout for writer configuration here.
+<ul>
+
+<li><tt>batchSize</tt> : Integer indicating number of records to batch together before sending to the writer. (default to <tt>15</tt>)</li>
+<li><tt>batchTimeout</tt> : The timeout after which a batch will be flushed even if batchSize has not been met. Optional. If unspecified, or set to <tt>0</tt>, it defaults to a system-determined duration which is a fraction of the Storm parameter <tt>topology.message.timeout.secs</tt>. Ignored if batchSize is <tt>1</tt>, since this disables batching.</li>
+<li>The kafka writer can be configured within the parser config as well. (This is all configured a priori, but this is convenient for overriding the settings). See <a href="../../metron-writer/index.html#kafka-writer">here</a></li>
+</ul>
+</li>
+<li><tt>fieldTransformations</tt> : An array of complex objects representing the transformations to be done on the message generated from the parser before writing out to the kafka topic.</li>
+<li><tt>securityProtocol</tt> : The security protocol to use for reading from kafka (this is a string). This can be overridden on the command line and also specified in the spout config via the <tt>security.protocol</tt> key. If both are specified, then they are merged and the CLI will take precedence. If multiple sensors are used, any non “PLAINTEXT” value will be used.</li>
+<li><tt>cacheConfig</tt> : Cache config for stellar field transformations. This configures a least frequently used cache. This is a map with the following keys. If not explicitly configured (the default), then no cache will be used.
+<ul>
+
+<li><tt>stellar.cache.maxSize</tt> - The maximum number of elements in the cache. Default is to not use a cache.</li>
+<li><tt>stellar.cache.maxTimeRetain</tt> - The maximum amount of time an element is kept in the cache (in minutes). Default is to not use a cache.
+<p>Example of a cache config to contain at max <tt>20000</tt> stellar expressions for at most <tt>20</tt> minutes.:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "cacheConfig" : {
+ "stellar.cache.maxSize" : 20000,
+ "stellar.cache.maxTimeRetain" : 20
+ }
+}
+</pre></div></div>
+</li>
+</ul>
+</li>
+</ul>
+<p>The <tt>fieldTransformations</tt> is a complex object which defines a transformation which can be done to a message. This transformation can</p>
+<ul>
+
+<li>Modify existing fields to a message</li>
+<li>Add new fields given the values of existing fields of a message</li>
+<li>Remove existing fields of a message</li>
+</ul>
+<p>For platform specific configs, see the README of the appropriate project. This would include settings such as parallelism of individual components and general configuration.</p>
+<ul>
+
+<li><a href="metron-parsing-storm/index.html#parser-configuration">Storm</a></li>
+</ul>
+<div class="section">
+<h3><a name="Metadata"></a>Metadata</h3>
+<p>Metadata is a useful thing to send to Metron and use during enrichment or threat intelligence.<br />
+Consider the following scenarios:</p>
+<ul>
+
+<li>You have multiple telemetry sources of the same type that you want to
+<ul>
+
+<li>ensure downstream analysts can differentiate</li>
+<li>ensure profiles consider independently as they have different seasonality or some other fundamental characteristic</li>
+</ul>
+</li>
+</ul>
+<p>As such, there are two types of metadata that we seek to support in Metron:</p>
+<ul>
+
+<li>Environmental metadata : Metadata about the system at large
+<ul>
+
+<li>Consider the possibility that you have multiple kafka topics being processed by one parser and you want to tag the messages with the kafka topic</li>
+<li>At the moment, only the kafka topic is kept as the field name.</li>
+</ul>
+</li>
+<li>Custom metadata: Custom metadata from an individual telemetry source that one might want to use within Metron.</li>
+</ul>
+<p>Metadata is controlled by the following parser configs:</p>
+<ul>
+
+<li><tt>rawMessageStrategy</tt> : This is a strategy which indicates how to read data and metadata. The strategies supported are:
+<ul>
+
+<li><tt>DEFAULT</tt> : Data is read directly from the kafka record value and metadata, if any, is read from the kafka record key. This strategy defaults to not reading metadata and not merging metadata. This is the default strategy.</li>
+<li><tt>ENVELOPE</tt> : Data from kafka record value is presumed to be a JSON blob. One of these fields must contain the raw data to pass to the parser. All other fields should be considered metadata. The field containing the raw data is specified in the <tt>rawMessageStrategyConfig</tt>. Data held in the kafka key as well as the non-data fields in the JSON blob passed into the kafka value are considered metadata. Note that the exception to this is that any <tt>original_string</tt> field is inherited from the envelope data so that the original string contains the envelope data. If you do not prefer this behavior, remove this field from the envelope data.</li>
+</ul>
+</li>
+<li><tt>rawMessageStrategyConfig</tt> : The configuration (a map) for the <tt>rawMessageStrategy</tt>. Available configurations are strategy dependent:
+<ul>
+
+<li><tt>DEFAULT</tt>
+<ul>
+
+<li><tt>metadataPrefix</tt> defines the key prefix for metadata (default is <tt>metron.metadata</tt>).</li>
+</ul>
+</li>
+<li><tt>ENVELOPE</tt>
+<ul>
+
+<li><tt>metadataPrefix</tt> defines the key prefix for metadata (default is <tt>metron.metadata</tt>)</li>
+<li><tt>messageField</tt> defines the field from the envelope to use as the data. All other fields are considered metadata.</li>
+</ul>
+</li>
+</ul>
+</li>
+<li><tt>readMetadata</tt> : This is a boolean indicating whether metadata will be read and made available to Field transformations (i.e. Stellar field transformations). The default is dependent upon the <tt>rawMessageStrategy</tt>:
+<ul>
+
+<li><tt>DEFAULT</tt> : default to <tt>false</tt>.</li>
+<li><tt>ENVELOPE</tt> : default to <tt>true</tt>.</li>
+</ul>
+</li>
+<li><tt>mergeMetadata</tt> : This is a boolean indicating whether metadata fields will be merged with the message automatically. That is to say, if this property is set to <tt>true</tt> then every metadata field will become part of the messages and, consequently, also available for use in field transformations. The default is dependent upon the <tt>rawMessageStrategy</tt>:
+<ul>
+
+<li><tt>DEFAULT</tt> : default to <tt>false</tt>.</li>
+<li><tt>ENVELOPE</tt> : default to <tt>true</tt>.</li>
+</ul>
+</li>
+</ul>
+<div class="section">
+<h4><a name="Field_Naming"></a>Field Naming</h4>
+<p>In order to avoid collisions from metadata fields, metadata fields will be prefixed (the default is <tt>metron.metadata.</tt>, but this is configurable in the <tt>rawMessageStrategyConfig</tt>). So, for instance the kafka topic would be in the field <tt>metron.metadata.topic</tt>.</p></div>
+<div class="section">
+<h4><a name="Specifying_Custom_Metadata"></a>Specifying Custom Metadata</h4>
+<p>Custom metadata is specified by sending a JSON Map in the key. If no key is sent, then, obviously, no metadata will be parsed. For instance, sending a metadata field called <tt>customer_id</tt> could be done by sending</p>
+
+<div>
+<div>
+<pre class="source">{
+"customer_id" : "my_customer_id"
+}
+</pre></div></div>
+
+<p>in the kafka key. This would be exposed as the field <tt>metron.metadata.customer_id</tt> to stellar field transformations as well, if <tt>mergeMetadata</tt> is <tt>true</tt>, available as a field in its own right.</p></div>
+<div class="section">
+<h4><a name="Metadata_and_Error_Handling"></a>Metadata and Error Handling</h4>
+<p>When a telemetry message fails to parse correctly, a separate error message is produced and sent to the error topic. This error message will contain detailed information to reflect the error that occurred.</p>
+<p>If the telemetry message that failed contains metadata, this metadata is included in the error message. For example, here is an error message that contains two metadata fields; <tt>metron.metadata.topic</tt> and <tt>metron.metadata.customer</tt>.</p>
+
+<div>
+<div>
+<pre class="source">{
+ "exception": "java.lang.IllegalStateException: Unable to parse Message: \"this is an invalid synthetic message\" }",
+ "stack": "java.lang.IllegalStateException: Unable to parse Message: \"this is an invalid synthetic message\" ...\n",
+ "raw_message": "\"this is an invalid synthetic message\" }",
+ "error_hash": "3d498968e8df7f28d05db3037d4ad2a3a0095c22c14d881be45fac3f184dbcc3",
+ "message": "Unable to parse Message: \"this is an invalid synthetic message\" }",
+ "source.type": "error",
+ "failed_sensor_type": "bro",
+ "hostname": "node1",
+ "error_type": "parser_error",
+ "guid": "563d8d2a-1493-4758-be2f-5613bfd2d615",
+ "timestamp": 1548366516634,
+ "metron.metadata.topic": "bro",
+ "metron.metadata.customer": "acme-inc"
+}
+</pre></div></div>
+
+<p>By default, error messages are sent to the <tt>indexing</tt> topic. This will cause the errors to be indexed in whichever endpoints you have configured, namely Solr, Elasticsearch, and HDFS. You may need to update your configuration of these endpoints to accurately reflect the metadata fields contained in the error message. For example, you may need to update the schema definition of your Solr Collection for the metadata fields to be accurately indexed in the Error collection.</p></div></div>
+<div class="section">
+<h3><a name="fieldTransformation_configuration"></a><tt>fieldTransformation</tt> configuration</h3>
+<p>The format of a <tt>fieldTransformation</tt> is as follows:</p>
+<ul>
+
+<li><tt>input</tt> : An array of fields or a single field representing the input. This is optional; if unspecified, then the whole message is passed as input.</li>
+<li><tt>output</tt> : The outputs to produce from the transformation. If unspecified, it is assumed to be the same as inputs.</li>
+<li><tt>transformation</tt> : The fully qualified classname of the transformation to be used. This is either a class which implements <tt>FieldTransformation</tt> or a member of the <tt>FieldTransformations</tt> enum.</li>
+<li><tt>config</tt> : A String to Object map of transformation specific configuration.</li>
+</ul>
+<p>The currently implemented fieldTransformations are:</p>
+<ul>
+
+<li>
+
+<p><tt>REMOVE</tt> : This transformation removes the specified input fields. If you want a conditional removal, you can pass a Metron Query Language statement to define the conditions under which you want to remove the fields.</p>
+<p>Consider the following simple configuration which will remove <tt>field1</tt> unconditionally:</p>
+
+<div>
+<div>
+<pre class="source">{
+...
+ "fieldTransformations" : [
+ {
+ "input" : "field1"
+ , "transformation" : "REMOVE"
+ }
+ ]
+}
+</pre></div></div>
+
+<p>Consider the following simple sensor parser configuration which will remove <tt>field1</tt> whenever <tt>field2</tt> exists and whose corresponding equal to ‘foo’:</p>
+
+<div>
+<div>
+<pre class="source">{
+...
+ "fieldTransformations" : [
+ {
+ "input" : "field1"
+ , "transformation" : "REMOVE"
+ , "config" : {
+ "condition" : "exists(field2) and field2 == 'foo'"
+ }
+ }
+ ]
+}
+</pre></div></div>
+</li>
+<li>
+
+<p><tt>SELECT</tt>: This transformation filters the fields in the message to include only the configured output fields, and drops any not explicitly included.</p>
+<p>For example:</p>
+
+<div>
+<div>
+<pre class="source">{
+...
+ "fieldTransformations" : [
+ {
+ "output" : ["field1", "field2" ]
+ , "transformation" : "SELECT"
+ }
+ ]
+}
+</pre></div></div>
+
+<p>when applied to a message containing keys field1, field2 and field3, will only output the first two. It is also worth noting that two standard fields - timestamp and original_source - will always be passed along whether they are listed in output or not, since they are considered core required fields.</p>
+</li>
+<li>
+
+<p><tt>IP_PROTOCOL</tt> : This transformation maps IANA protocol numbers to consistent string representations.</p>
+<p>Consider the following sensor parser config to map the <tt>protocol</tt> field to a textual representation of the protocol:</p>
+
+<div>
+<div>
+<pre class="source">{
+...
+ "fieldTransformations" : [
+ {
+ "input" : "protocol"
+ , "transformation" : "IP_PROTOCOL"
+ }
+ ]
+}
+</pre></div></div>
+
+<p>This transformation would transform <tt>{ "protocol" : 6, "source.type" : "bro", ... }</tt> into <tt>{ "protocol" : "TCP", "source.type" : "bro", ...}</tt></p>
+</li>
+<li>
+
+<p><tt>STELLAR</tt> : This transformation executes a set of transformations expressed as <a href="../../metron-common/index.html">Stellar Language</a> statements.</p>
+</li>
+<li>
+
+<p><tt>RENAME</tt> : This transformation allows users to rename a set of fields. Specifically, the config is presumed to be the mapping. The keys to the config are the existing field names and the values for the config map are the associated new field name.</p>
+<p>The following config will rename the fields <tt>old_field</tt> and <tt>different_old_field</tt> to <tt>new_field</tt> and <tt>different_new_field</tt> respectively:</p>
+
+<div>
+<div>
+<pre class="source">{
+...
+ "fieldTransformations" : [
+ {
+ "transformation" : "RENAME",
+ , "config" : {
+ "old_field" : "new_field",
+ "different_old_field" : "different_new_field"
+ }
+ }
+ ]
+}
+</pre></div></div>
+</li>
+<li>
+
+<p><tt>REGEX_SELECT</tt> : This transformation lets users set an output field to one of a set of possibilities based on matching regexes. This transformation is useful when the number or conditions are large enough to make a stellar language match statement unwieldy.</p>
+<p>The following config will set the field <tt>logical_source_type</tt> to one of the following, dependent upon the value of the <tt>pix_type</tt> field:</p>
+<ul>
+
+<li><tt>cisco-6-302</tt> if <tt>pix_type</tt> starts with either <tt>6-302</tt> or <tt>06-302</tt></li>
+<li><tt>cisco-5-304</tt> if <tt>pix_type</tt> starts with <tt>5-304</tt></li>
+</ul>
+
+<div>
+<div>
+<pre class="source">{
+...
+ "fieldTransformations" : [
+ {
+ "transformation" : "REGEX_ROUTING"
+ ,"input" : "pix_type"
+ ,"output" : "logical_source_type"
+ ,"config" : {
+ "cisco-6-302" : [ "^6-302.*", "^06-302.*"]
+ "cisco-5-304" : "^5-304.*"
+ }
+ }
+ ]
+...
+}
+</pre></div></div>
+</li>
+</ul></div>
+<div class="section">
+<h3><a name="Assignment_to_null"></a>Assignment to <tt>null</tt></h3>
+<p>If, in your field transformation, you assign a field to <tt>null</tt>, the field will be removed. You can use this capability to rename variables. It is preferred, however, that the <tt>RENAME</tt> field transformation is used in this situation as it is less awkward.</p>
+<p>Consider this example:</p>
+
+<div>
+<div>
+<pre class="source"> "fieldTransformations" : [
+ { "transformation" : "STELLAR"
+ ,"output" : [ "new_field", "old_field"]
+ ,"config" : {
+ "new_field" : "old_field"
+ ,"old_field" : "null"
+ }
+ }
+ ]
+</pre></div></div>
+
+<p>This would set <tt>new_field</tt> to the value of <tt>old_field</tt> and remove <tt>old_field</tt>.</p></div>
+<div class="section">
+<h3><a name="Warning:_Transforming_the_same_field_twice"></a>Warning: Transforming the same field twice</h3>
+<p>Currently, the stellar expressions are expressed in the form of a map where the keys define the fields and the values define the Stellar expressions. You order the expression evaluation in the <tt>output</tt> field. A consequence of this choice to store the assignments as a map is that the same field cannot appear in the map as a key twice.</p>
+<p>For instance, the following will not function as expected:</p>
+
+<div>
+<div>
+<pre class="source"> "fieldTransformations" : [
+ { "transformation" : "STELLAR"
+ ,"output" : [ "new_field"]
+ ,"config" : {
+ "new_field" : "TO_UPPER(field1)"
+ ,"new_field" : "TO_LOWER(new_field)"
+ }
+ }
+ ]
+</pre></div></div>
+
+<p>In the above example, the last instance of <tt>new_field</tt> will win and <tt>TO_LOWER(new_field)</tt> will be evaluated while <tt>TO_UPPER(field1)</tt> will be skipped.</p></div>
+<div class="section">
+<h3><a name="Example"></a>Example</h3>
+<p>Consider the following sensor parser config to add three new fields to a message:</p>
+<ul>
+
+<li><tt>utc_timestamp</tt> : The unix epoch timestamp based on the <tt>timestamp</tt> field, a <tt>dc</tt> field which is the data center the message comes from and a <tt>dc2tz</tt> map mapping data centers to timezones</li>
+<li><tt>url_host</tt> : The host associated with the url in the <tt>url</tt> field</li>
+<li><tt>url_protocol</tt> : The protocol associated with the url in the <tt>url</tt> field</li>
+</ul>
+
+<div>
+<div>
+<pre class="source">{
+...
+ "fieldTransformations" : [
+ {
+ "transformation" : "STELLAR"
+ ,"output" : [ "utc_timestamp", "url_host", "url_protocol" ]
+ ,"config" : {
+ "utc_timestamp" : "TO_EPOCH_TIMESTAMP(timestamp, 'yyyy-MM-dd
+HH:mm:ss', MAP_GET(dc, dc2tz, 'UTC') )"
+ ,"url_host" : "URL_TO_HOST(url)"
+ ,"url_protocol" : "URL_TO_PROTOCOL(url)"
+ }
+ }
+ ]
+ ,"parserConfig" : {
+ "dc2tz" : {
+ "nyc" : "EST"
+ ,"la" : "PST"
+ ,"london" : "UTC"
+ }
+ }
+}
+</pre></div></div>
+
+<p>Note that the <tt>dc2tz</tt> map is in the parser config, so it is accessible in the functions.</p></div>
+<div class="section">
+<h3><a name="An_Example_Configuration_for_a_Sensor"></a>An Example Configuration for a Sensor</h3>
+<p>Consider the following example configuration for the <tt>yaf</tt> sensor:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "parserClassName":"org.apache.metron.parsers.GrokParser",
+ "sensorTopic":"yaf",
+ "fieldTransformations" : [
+ {
+ "input" : "protocol"
+ ,"transformation": "IP_PROTOCOL"
+ }
+ ],
+ "parserConfig":
+ {
+ "grokPath":"/patterns/yaf",
+ "patternLabel":"YAF_DELIMITED",
+ "timestampField":"start_time",
+ "timeFields": ["start_time", "end_time"],
+ "dateFormat":"yyyy-MM-dd HH:mm:ss.S"
+ }
+}
+</pre></div></div>
+</div></div>
+<div class="section">
+<h2><a name="Parser_Adapters"></a>Parser Adapters</h2>
+<p>Parser adapters are loaded dynamically in each Metron topology. They are defined in the Parser Config (defined above) JSON file in Zookeeper.</p>
+<div class="section">
+<h3><a name="Java_Parser_Adapters"></a>Java Parser Adapters</h3>
+<p>Java parser adapters are intended for higher-velocity topologies and are not easily changed or extended. As the adoption of Metron continues we plan on extending our library of Java adapters to process more log formats. As of this moment the Java adapters included with Metron are:</p>
+<ul>
+
+<li>BasicIseParser : Parse ISE messages</li>
+<li>org.apache.metron.parsers.bro.BasicBroParser : Parse Bro messages</li>
+<li>org.apache.metron.parsers.sourcefire.BasicSourcefireParser : Parse Sourcefire messages</li>
+<li>org.apache.metron.parsers.lancope.BasicLancopeParser : Parse Lancope messages</li>
+<li>org.apache.metron.parsers.syslog.Syslog5424Parser : Parse Syslog RFC 5424 messages</li>
+<li>org.apache.metron.parsers.syslog.Syslog3164Parser : Parse Syslog RFC 3164 messages</li>
+</ul></div>
+<div class="section">
+<h3><a name="Grok_Parser_Adapters"></a>Grok Parser Adapters</h3>
+<p>Grok parser adapters are designed primarily for someone who is not a Java coder for quickly standing up a parser adapter for lower velocity topologies. Grok relies on Regex for message parsing, which is much slower than purpose-built Java parsers, but is more extensible. Grok parsers are defined via a config file and the topplogy does not need to be recompiled in order to make changes to them. Example of a Grok parsers are:</p>
+<ul>
+
+<li>org.apache.metron.parsers.GrokParser and org.apache.metron.parsers.websphere.GrokWebSphereParser</li>
+</ul>
+<p>Parsers that derive from GrokParser typically allow the GrokParser to parse the messages, and then override the methods for postParse to do further parsing. When this is the case, and the Parser has not overridden <tt>parse(byte[])</tt> or <tt>parseResultOptional(byte[])</tt> these parsers will gain support for treating byte[] input as multiple lines, with each line parsed as a separate message ( and returned as such). This is enabled by using the <tt>"multiline":"true"</tt> Parser configuration option.</p>
+<p>For more information on the Grok project please refer to the following link:</p>
+<p><a class="externalLink" href="https://github.com/thekrakken/java-grok">https://github.com/thekrakken/java-grok</a></p>
+<p><a name="Starting_the_Parser"></a></p>
+<h1>Starting the Parser</h1>
+<p>Starting a particular parser on a running Metron deployment is dependent on the platform being run on. Please see the appropriate platform-specific README.</p>
+<ul>
+
+<li><a href="metron-parsing-storm/index.html#starting-the-parser-topology">Storm</a></li>
+</ul>
+<p>For all platforms, you will need to provide</p>
+<ul>
+
+<li>Zookeeper Quorum</li>
+<li>Kafka Broker URL</li>
+<li>Sensor type</li>
+<li>Output topic</li>
+<li>Kakfa Security Protocol (Optional)</li>
+</ul>
+<p><a name="Notes_on_Performance_Tuning"></a></p>
+<h1>Notes on Performance Tuning</h1>
+<p>Default installed Metron is untuned for production deployment. There are a few knobs to tune to get the most out of your system.</p>
+<p>When using aggregated parsers, it’s highly recommended to aggregate parsers with similar velocity and parser complexity together.</p>
+<p>Platform specific notes can be found in the appropriate README</p>
+<ul>
+
+<li><a href="metron-parsing-storm/index.html">Storm</a></li>
+</ul>
+<p><a name="Notes_on_Adding_a_New_Sensor"></a></p>
+<h1>Notes on Adding a New Sensor</h1>
+<p>In order to allow for meta alerts to be queries alongside regular alerts in Elasticsearch 2.x, it is necessary to add an additional field to the templates and mapping for existing sensors.</p>
+<p>Please see a description of the steps necessary to make this change in the metron-elasticsearch <a href="../../metron-elasticsearch/index.html#Using_Metron_with_Elasticsearch_2.x">Using Metron with Elasticsearch 2.x</a></p>
+<p>If Solr is selected as the real-time store, it is also necessary to add additional fields. See the <a href="../../metron-indexing/index.html#Solr">Solr</a> section in metron-indexing for more details.</p></div></div>
+<div class="section">
+<h2><a name="Kafka_Queue"></a>Kafka Queue</h2>
+<p>The kafka queue associated with your parser is a collection point for all of the data sent to your parser. As such, make sure that the number of partitions in the kafka topic is sufficient to handle the throughput that you expect from your parser topology.</p></div>
+<div class="section">
+<h2><a name="JSON_Path"></a>JSON Path</h2>
+<blockquote>
+
+<dl>
+
+<dt>“JSONPath expressions always refer to a JSON structure in the same way as XPath expression are used in combination with an XML document.”</dt>
+<dd>Stefan Goessner</dd>
+</dl>
+</blockquote>
+<ul>
+
+<li><a class="externalLink" href="http://goessner.net/articles/JsonPath/">JSON Path concept</a></li>
+<li><a class="externalLink" href="https://github.com/json-path/JsonPath">Read about JSON Path library Apache Metron uses</a></li>
+<li><a class="externalLink" href="http://jsonpath.herokuapp.com">Try JSON Path expressions online</a></li>
+</ul></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.html b/current-book/metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.html
new file mode 100644
index 0000000..3ee61d5
--- /dev/null
+++ b/current-book/metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.html
@@ -0,0 +1,491 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Custom Metron Parsers</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">Custom Metron Parsers</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-down"></span>Parsing</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/index.html" title="Parsers-common"><span class="icon-chevron-down"></span>Parsers-common</a>
+ <ul class="nav nav-list">
+ <li class="active"><a href="#"><span class="none"></span>3rdPartyParser</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/ParserChaining.html" title="ParserChaining"><span class="none"></span>ParserChaining</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/message-parser-implementation-notes.html" title="message-parser-implementation-notes"><span class="none"></span>message-parser-implementation-notes</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/parser-testing.html" title="parser-testing"><span class="none"></span>parser-testing</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/paloalto/index.html" title="Paloalto"><span class="none"></span>Paloalto</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsing-storm/index.html" title="Parsing-storm"><span class="none"></span>Parsing-storm</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Custom Metron Parsers</h1>
+<p><a name="Custom_Metron_Parsers"></a></p>
+<p>We have many stock parsers for normal operations. Some of these are networking and cybersecurity focused (e.g. the ASA Parser), some of these are general purpose (e.g. the CSVParser), but inevitably users will want to extend the system to process their own data formats. To enable this, this is a walkthrough of how to create and use a custom parser within Metron.</p>
+<p><a name="Writing_A_Custom_Parser"></a></p>
+<h1>Writing A Custom Parser</h1>
+<p>Before we can use a parser, we will need to create a custom parser. The parser is the workhorse of Metron ingest. It provides the mapping between the raw data coming in via the Kafka value and a <tt>JSONObject</tt>, the internal data structure provided.</p>
+<div class="section">
+<h2><a name="Implementation"></a>Implementation</h2>
+<p>In order to do create a custom parser, we need to do one of the following:</p>
+<ul>
+
+<li>Write a class which conforms to the <tt>org.apache.metron.parsers.interfaces.MessageParser<JSONObject></tt> and <tt>java.util.Serializable</tt> interfaces
+<ul>
+
+<li>Implement <tt>init()</tt>, <tt>validate(JSONObject message)</tt>, and <tt>List<JSONObject> parse(byte[] rawMessage)</tt></li>
+</ul>
+</li>
+<li>Write a class which extends <tt>org.apache.metron.parsers.BasicParser</tt>
+<ul>
+
+<li>Provides convenience implementations to <tt>validate</tt> which ensures <tt>timestamp</tt> and <tt>original_string</tt> fields exist.</li>
+</ul>
+</li>
+</ul></div>
+<div class="section">
+<h2><a name="Example"></a>Example</h2>
+<p>In order to illustrate how this might be done, let’s create a very simple parser that takes a comma separated pair and creates a couple of fields:</p>
+<ul>
+
+<li><tt>original_string</tt> – the raw data</li>
+<li><tt>timestamp</tt> – the current time</li>
+<li><tt>first</tt> – the first field of the comma separated pair</li>
+<li><tt>last</tt> – the last field of the comma separated pair</li>
+</ul>
+<p>For this demonstration, let’s create a maven project to compile our project. We’ll call it <tt>extra_parsers</tt>, so in your workspace, let’s set up the maven project:</p>
+<ul>
+
+<li>
+
+<p>Create the maven infrastructure for <tt>extra_parsers</tt> via</p>
+
+<div>
+<div>
+<pre class="source">mkdir -p extra_parsers/src/{main,test}/java
+</pre></div></div>
+</li>
+<li>
+
+<p>Create a pom file indicating how we should build our parsers by editing <tt>extra_parsers/pom.xml</tt> with the following content:</p>
+
+<div>
+<div>
+<pre class="source"><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>com.3rdparty</groupId>
+ <artifactId>extra-parsers</artifactId>
+ <packaging>jar</packaging>
+ <version>1.0-SNAPSHOT</version>
+ <name>extra-parsers</name>
+ <url>http://thirdpartysoftware.org</url>
+ <properties>
+ <!-- The java version to conform to. Metron works all the way to 1.8 -->
+ <java_version>1.8</java_version>
+ <!-- The version of Metron that we'll be targetting. -->
+ <metron_version>0.4.1</metron_version>
+ <!-- To complete the simulation, we'll depend on a common dependency -->
+ <guava_version>19.0</guava_version>
+ <!-- We will shade our dependencies to create a single jar at the end -->
+ <shade_version>2.4.3</shade_version>
+ </properties>
+ <dependencies>
+ <!--
+ We want to depend on Metron, but ensure that the scope is "provided"
+ as we do not want to include it in our bundle.
+ -->
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-parsers-common</artifactId>
+ <version>${metron_version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>3.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <!-- We will set up the shade plugin to create a single jar at the
+ end of the build lifecycle. We will exclude some things and
+ relocate others to simulate a real situation.
+
+ One thing to note is that it's a good practice to shade and
+ relocate common libraries that may be dependencies in Metron.
+ Your jar will be merged with the parsers jar, so the metron
+ version will be included for all overlapping classes.
+ So, shade and relocate to ensure that YOUR version of the library is used.
+ -->
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <artifactSet>
+ <excludes>
+ <!-- Exclude slf4j for no reason other than to illustrate how to exclude dependencies.
+ The metron team has nothing against slf4j. :-)
+ -->
+ <exclude>*slf4j*</exclude>
+ </excludes>
+ </artifactSet>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>uber</shadedClassifierName>
+ <filters>
+ <filter>
+ <!-- Sometimes these get added and confuse the uber jar out of shade -->
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <!-- Relocate guava as it's used in Metron and I really want 0.19 -->
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>com.thirdparty.guava</shadedPattern>
+ </relocation>
+ </relocations>
+ <artifactSet>
+ <excludes>
+ <!-- We can also exclude by artifactId and groupId -->
+ <exclude>storm:storm-core:*</exclude>
+ <exclude>storm:storm-lib:*</exclude>
+ <exclude>org.slf4j.impl*</exclude>
+ <exclude>org.slf4j:slf4j-log4j*</exclude>
+ </excludes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!--
+ We want to make sure we compile using java 1.8.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.5.1</version>
+ <configuration>
+ <forceJavacCompilerUse>true</forceJavacCompilerUse>
+ <source>${java_version}</source>
+ <compilerArgument>-Xlint:unchecked</compilerArgument>
+ <target>${java_version}</target>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
+</pre></div></div>
+</li>
+<li>
+
+<p>Now let’s create our parser <tt>com.thirdparty.SimpleParser</tt> by creating the file <tt>extra-parsers/src/main/java/com/thirdparty/SimpleParser.java</tt> with the following content:</p>
+
+<div>
+<div>
+<pre class="source">package com.thirdparty;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+
+import java.util.List;
+import java.util.Map;
+
+public class SimpleParser extends BasicParser {
+ @Override
+ public void init() {
+
+ }
+
+ @Override
+ public List<JSONObject> parse(byte[] bytes) {
+ String input = new String(bytes);
+ Iterable<String> it = Splitter.on(",").split(input);
+ JSONObject ret = new JSONObject();
+ ret.put("original_string", input);
+ ret.put("timestamp", System.currentTimeMillis());
+ ret.put("first", Iterables.getFirst(it, "missing"));
+ ret.put("last", Iterables.getLast(it, "missing"));
+ return ImmutableList.of(ret);
+ }
+
+ @Override
+ public void configure(Map<String, Object> map) {
+
+ }
+}
+</pre></div></div>
+</li>
+<li>Compile the parser via <tt>mvn clean package</tt> in <tt>extra_parsers</tt></li>
+<li>
+
+<p>This will create a jar containing your parser and its dependencies (sans Metron dependencies) in <tt>extra-parsers/target/extra-parsers-1.0-SNAPSHOT-uber.jar</tt></p>
+</li>
+</ul>
+<p><a name="Deploying_Your_Custom_Parser"></a></p>
+<h1>Deploying Your Custom Parser</h1>
+<p>In order to deploy your newly built custom parser, you would place the jar file above in the <tt>$METRON_HOME/parser_contrib</tt> directory on the Metron host (i.e. any host you would start parsers from or, alternatively, where the Metron REST is hosted).</p></div>
+<div class="section">
+<h2><a name="Example"></a>Example</h2>
+<p>Let’s work through deploying the example above.</p>
+<div class="section">
+<h3><a name="Preliminaries"></a>Preliminaries</h3>
+<p>We assume that the following environment variables are set:</p>
+<ul>
+
+<li><tt>METRON_HOME</tt> - the home directory for metron</li>
+<li><tt>ZOOKEEPER</tt> - The zookeeper quorum (comma separated with port specified: e.g. <tt>node1:2181</tt> for full-dev)</li>
+<li><tt>BROKERLIST</tt> - The Kafka broker list (comma separated with port specified: e.g. <tt>node1:6667</tt> for full-dev)</li>
+<li><tt>ES_HOST</tt> - The elasticsearch master (and port) e.g. <tt>node1:9200</tt> for full-dev.</li>
+</ul>
+<p>Also, this does not assume that you are using a kerberized cluster. If you are, then the parser start command will adjust slightly to include the security protocol.</p></div>
+<div class="section">
+<h3><a name="Copy_the_jar_file_up"></a>Copy the jar file up</h3>
+<p>Copy the jar file located in <tt>extra-parsers/target/extra-parsers-1.0-SNAPSHOT-uber.jar</tt> to <tt>$METRON_HOME/parser_contrib</tt> and ensure the permissions are such that the <tt>metron</tt> user can read and execute.</p></div>
+<div class="section">
+<h3><a name="Restart_the_REST_service_in_Ambari"></a>Restart the REST service in Ambari</h3>
+<p>In order for new parsers to be picked up, the REST service must be restarted. You can do that from within Ambari by restarting the <tt>Metron REST</tt> service.</p></div>
+<div class="section">
+<h3><a name="Create_a_Kafka_Topic"></a>Create a Kafka Topic</h3>
+<p>Create a kafka topic, let’s call it <tt>test</tt>.</p>
+
+<div>
+<div>
+<pre class="source">KAFKA_HOME=/usr/hdp/current/kafka-broker
+$KAFKA_HOME/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic test --partitions 1 --replication-factor 1
+</pre></div></div>
+
+<p>Note, in a real deployment, that topic would be named something more descriptive and would have replication factor and partitions set to something less trivial.</p></div>
+<div class="section">
+<h3><a name="Configure_Test_Parser"></a>Configure Test Parser</h3>
+<p>Create the a file called <tt>$METRON_HOME/config/zookeeper/parsers/test.json</tt> with the following content:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "parserClassName":"com.thirdparty.SimpleParser",
+ "sensorTopic":"test"
+}
+</pre></div></div>
+</div>
+<div class="section">
+<h3><a name="Push_the_Zookeeper_Configs"></a>Push the Zookeeper Configs</h3>
+<p>Now push the config to Zookeeper with the following command.</p>
+
+<div>
+<div>
+<pre class="source">$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i $METRON_HOME/config/zookeeper/ -z $ZOOKEEPER
+</pre></div></div>
+</div>
+<div class="section">
+<h3><a name="Start_Parser"></a>Start Parser</h3>
+<p>Now we can start the parser and send some data through:</p>
+<ul>
+
+<li>
+
+<p>Start the parser</p>
+
+<div>
+<div>
+<pre class="source">$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s test
+</pre></div></div>
+</li>
+<li>
+
+<p>Send example data through:</p>
+
+<div>
+<div>
+<pre class="source">echo "apache,metron" | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic test
+</pre></div></div>
+</li>
+<li>
+
+<p>Validate data was written in ES:</p>
+
+<div>
+<div>
+<pre class="source">curl -XPOST "http://$ES_HOST/test*/_search?pretty" -d '
+{
+ "_source" : [ "original_string", "timestamp", "first", "last"]
+}
+'
+</pre></div></div>
+</li>
+<li>
+
+<p>This should yield something like:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "took" : 23,
+ "timed_out" : false,
+ "_shards" : {
+ "total" : 1,
+ "successful" : 1,
+ "failed" : 0
+ },
+ "hits" : {
+ "total" : 1,
+ "max_score" : 1.0,
+ "hits" : [ {
+ "_index" : "test_index_2017.10.04.17",
+ "_type" : "test_doc",
+ "_id" : "3ae4dd4d-8c09-4f2a-93c0-26ec5508baaa",
+ "_score" : 1.0,
+ "_source" : {
+ "original_string" : "apache,metron",
+ "last" : "metron",
+ "first" : "apache",
+ "timestamp" : 1507138373223
+ }
+ } ]
+ }
+}
+</pre></div></div>
+</li>
+</ul></div>
+<div class="section">
+<h3><a name="Via_the_Management_UI"></a>Via the Management UI</h3>
+<p>As long as the REST service is restarted after new parsers are added to <tt>$METRON_HOME/parser_contrib</tt>, they are available in the UI to creating and deploying parsers.</p></div></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-parsing/metron-parsers-common/ParserChaining.html b/current-book/metron-platform/metron-parsing/metron-parsers-common/ParserChaining.html
new file mode 100644
index 0000000..a5ac158
--- /dev/null
+++ b/current-book/metron-platform/metron-parsing/metron-parsers-common/ParserChaining.html
@@ -0,0 +1,285 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsing/metron-parsers-common/ParserChaining.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Parser Chaining</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">Parser Chaining</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-down"></span>Parsing</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/index.html" title="Parsers-common"><span class="icon-chevron-down"></span>Parsers-common</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.html" title="3rdPartyParser"><span class="none"></span>3rdPartyParser</a></li>
+ <li class="active"><a href="#"><span class="none"></span>ParserChaining</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/message-parser-implementation-notes.html" title="message-parser-implementation-notes"><span class="none"></span>message-parser-implementation-notes</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/parser-testing.html" title="parser-testing"><span class="none"></span>parser-testing</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/paloalto/index.html" title="Paloalto"><span class="none"></span>Paloalto</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsing-storm/index.html" title="Parsing-storm"><span class="none"></span>Parsing-storm</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Parser Chaining</h1>
+<p><a name="Parser_Chaining"></a></p>
+<p>Aggregating many different types sensors into a single data source (e.g. syslog) and ingesting that aggregate sensor into Metron is a common pattern. It is not obvious precisely how to manage these types of aggregate sensors as they require two-pass parsing. This document will walk through an example of supporting this kind of multi-pass ingest.</p>
+<p>Multi-pass parser involves the following requirements:</p>
+<ul>
+
+<li>The enveloping parser (e.g. the aggregation format such as syslog or plain CSV) may contain metadata which should be ingested along with the data.</li>
+<li>The enveloping sensor contains many different sensor types</li>
+</ul>
+<p><a name="High_Level_Solution"></a></p>
+<h1>High Level Solution</h1>
+<p><img src="../../../images/message_routing_high_level.svg" alt="High Level Approach" /></p>
+<p>At a high level, we continue to maintain the architectural invariant of a 1-1 relationship between logical sensors and storm topologies. Eventually this relationship may become more complex, but at the moment the approach is to construct a routing parser which will have two responsibilities:</p>
+<ul>
+
+<li>Parse the envelope (e.g. syslog data) and extract any metadata fields from the envelope to pass along</li>
+<li>Route the unfolded data to the appropriate kafka topic associated with the enveloped sensor data</li>
+</ul>
+<p>Because the data emitted from the routing parser is just like any data emitted from any other parser, in that it is a JSON blob like any data emitted from any parser, we will need to adjust the downstream parsers to extract the enveloped data from the JSON blob and treat it as the data to parse.</p>
+<p><a name="Aggregated_Parsers_with_Parser_Chaining"></a></p>
+<h1>Aggregated Parsers with Parser Chaining</h1>
+<p>Chained parsers can be run as aggregated parsers. These parsers continue to use the sensor specific Kafka topics, and do not do internal routing to the appropriate sensor.</p>
+<p>Say, there were three sensors (<tt>bro</tt>, <tt>snort</tt> and <tt>yaf</tt>). Instead of creating a topology per sensor, all 3 can be run in a single aggregated parser. It is also possible to aggregate a subset of these parsers (e.g. run <tt>bro</tt> as it’s own topology, and aggregate the other 2).</p>
+<p>The step to start an aggregated parsers then becomes</p>
+
+<div>
+<div>
+<pre class="source">$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s bro,snort,yaf
+</pre></div></div>
+
+<p>which will result in a single storm topology named <tt>bro__snort__yaf</tt> to run.</p>
+<p>Aggregated parsers can be specified using the Ambari Metron config as well under Services -> Metron -> Configs -> ‘Parsers’ tab -> ‘Metron Parsers’ field. The grouping is configured by enclosing the desired parsers in double quotes.</p>
+<p>Some examples of specifying aggregated parsers are as follows:</p>
+<ul>
+
+<li>“bro,snort,yaf” –> Will start a single topology named <tt>bro__snort__yaf</tt></li>
+<li>“ciscopixA,ciscopixB”,yaf,“squid,ciscopixC” –> Will start three topologies viz. <tt>ciscopixA__ciscopixB</tt>, <tt>yaf</tt> and <tt>squid__ciscopixC</tt></li>
+</ul>
+<p><a name="Architecting_a_Parser_Chaining_Solution_in_Metron"></a></p>
+<h1>Architecting a Parser Chaining Solution in Metron</h1>
+<p>Currently the approach to fulfill this requirement involves a couple knobs in the Parser infrastructure for Metron.</p>
+<p>Consider the case, for instance, where we have many different TYPES of messages wrapped inside of syslog. As an architectural abstraction, we would want to have the following properties:</p>
+<ul>
+
+<li>separate the concerns of parsing the individual types of messages from each other</li>
+<li>separate the concerns of parsing the individual types of messages from parsing the envelope</li>
+</ul>
+<div class="section">
+<h2><a name="Data_Dependent_Parser_Writing"></a>Data Dependent Parser Writing</h2>
+<p>Parsers allow users to configure the topic which the kafka producer uses in a couple of ways (from the parser config in an individual parser):</p>
+<ul>
+
+<li><tt>kafka.topic</tt> - Specify the topic in the config. This can be updated by updating the config, but it is data independent (e.g. not dependent on the data in a message).</li>
+<li><tt>kafka.topicField</tt> - Specify the topic as the value of a particular field. If unpopulated, then the message is dropped. This is inherrently data dependent.</li>
+</ul>
+<p>The <tt>kafka.topicField</tt> parameter allows for data dependent topic selection and this inherrently enables the routing capabilities necessary for handling enveloped data.</p></div>
+<div class="section">
+<h2><a name="Flexibly_Interpreting_Data"></a>Flexibly Interpreting Data</h2>
+<div class="section">
+<h3><a name="Aside:_The_Role_of_Metadata_in_Metron"></a>Aside: The Role of Metadata in Metron</h3>
+<p>Before we continue, let’s briefly talk about metadata. We have exposed the ability to pass along metadata and interact with metadata in a decoupled way from the actual parser logic (i.e. the GrokParser should not have to consider how to interpret metadata).</p>
+<p>There are three choices about manipulating metadata in Metron:</p>
+<ul>
+
+<li>Should you merge metadata into the downstream message?</li>
+<li>If you do, should you use a key prefix to set it off from the message by default?</li>
+</ul>
+<p>This enables users to specify metadata independent of the data that is persisted downstream and can inform the operations of enrichment and the profiler.</p></div>
+<div class="section">
+<h3><a name="Interpretation"></a>Interpretation</h3>
+<p>Now that we have an approach which enables the routing of the data, the remaining question is how to decouple <i>parsing</i> data from <i>interpreting</i> data and metadata. By default, Metron operates like so:</p>
+<ul>
+
+<li>The kafka record key (as a JSON Map) is considered metadata</li>
+<li>The kafka record value is considered data</li>
+</ul>
+<p>Beyond that, we presume defaults for this default strategy around handling metadata. In particular, by default we do not merge metadata and use a <tt>metron.metadata</tt> prefix for all metadata.</p>
+<p>In order to enable chained parser WITH metadata, we allow the following to be specified via strategy in the parser config:</p>
+<ul>
+
+<li>How to extract the data from the kafka record</li>
+<li>How to extract the metadata from the kafka record</li>
+<li>The default operations for merging</li>
+<li>The prefix for the metadata key</li>
+</ul>
+<p>The available strategies, specified by the <tt>rawMessageStrategy</tt> configuration is either<tt>ENVELOPE</tt> or <tt>DEFAULT</tt>.</p>
+<p>Specifically, to enable parsing enveloped data (i.e. data in a field of a JSON blob with the other fields being metadata), one can specify the strategy and configuration of that strategy in the parser config. One must specify the <tt>rawMessageStrategy</tt> as <tt>ENVELOPE</tt> in the parser and the <tt>rawMessageStrategyConfig</tt> to indicate the field which contains the data.</p>
+<p>Together with routing, we have the complete solution to chain parsers which can:</p>
+<ul>
+
+<li>parse the envelope</li>
+<li>route the parsed data to specific parsers</li>
+<li>have the specific parsers interpret the data via the <tt>rawMessageStrategy</tt> whereby they pull the data out from JSON Map that they receive</li>
+</ul>
+<p>Together this enables a directed acyclic graph of parsers to handle single or multi-layer parsing.</p></div>
+<div class="section">
+<h3><a name="Example"></a>Example</h3>
+<p>For a complete example, look at the <a href="../../../use-cases/parser_chaining/index.html">parser chaining use-case</a>, however for a simple example the following should suffice.</p>
+<p>If I want to configure a CSV parser to parse data which has 3 columns <tt>f1</tt>, <tt>f2</tt> and <tt>f3</tt> and is held in a field called <tt>payload</tt> inside of a JSON Map, I can do so like this:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "parserClassName" : "org.apache.metron.parsers.csv.CSVParser"
+ ,"sensorTopic" : "my_topic"
+ ,"rawMessageStrategy" : "ENVELOPE"
+ ,"rawMessageStrategyConfig" : {
+ "messageField" : "payload",
+ "metadataPrefix" : ""
+ }
+ , "parserConfig": {
+ "columns" : { "f1": 0,
+ , "f2": 1,
+ , "f3": 2
+ }
+ }
+}
+</pre></div></div>
+
+<p>This would parse the following message:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "meta_f1" : "val1",
+ "payload" : "foo,bar,grok",
+ "original_string" : "2019 Jul, 01: val1 foo,bar,grok",
+ "timestamp" : 10000
+}
+</pre></div></div>
+
+<p>into</p>
+
+<div>
+<div>
+<pre class="source">{
+ "meta_f1" : "val1",
+ "f1" : "foo",
+ "f2" : "bar",
+ "f3" : "grok",
+ "original_string" : "2019 Jul, 01: val1 foo,bar,grok",
+ "timestamp" : 10002
+}
+</pre></div></div>
+
+<p>Note a couple of things here:</p>
+<ul>
+
+<li>The metadata field <tt>meta_f1</tt> is not prefixed here because we configured the strategy with <tt>metadataPrefix</tt> as empty string.</li>
+<li>The <tt>timestamp</tt> is not inherited from the metadata</li>
+<li>The <tt>original_string</tt> is inherited from the metadata</li>
+</ul></div></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-parsing/metron-parsers-common/index.html b/current-book/metron-platform/metron-parsing/metron-parsers-common/index.html
new file mode 100644
index 0000000..418cbec
--- /dev/null
+++ b/current-book/metron-platform/metron-parsing/metron-parsers-common/index.html
@@ -0,0 +1,145 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsing/metron-parsers-common/index.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – metron-parsers-common</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">metron-parsers-common</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-down"></span>Parsing</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li class="active"><a href="#"><span class="icon-chevron-down"></span>Parsers-common</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.html" title="3rdPartyParser"><span class="none"></span>3rdPartyParser</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/ParserChaining.html" title="ParserChaining"><span class="none"></span>ParserChaining</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/message-parser-implementation-notes.html" title="message-parser-implementation-notes"><span class="none"></span>message-parser-implementation-notes</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/parser-testing.html" title="parser-testing"><span class="none"></span>parser-testing</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/paloalto/index.html" title="Paloalto"><span class="none"></span>Paloalto</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsing-storm/index.html" title="Parsing-storm"><span class="none"></span>Parsing-storm</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>metron-parsers-common</h1>
+<p><a name="metron-parsers-common"></a></p>
+<p>metron-parsers-common contains the necessary components to write and test MessageParsers, along with some of the supporting utilities. The most basic parsers for extensions are included.</p>
+<p>The included parsers are</p>
+<ul>
+
+<li>Grok Parser</li>
+<li>JSONMapParser</li>
+<li>CSVParser</li>
+<li>Syslog 3164 and 5424 parsers</li>
+</ul>
+<p>More details on these parsers and the overall architecture can be found in the metron-parsing <a href="..#README.md">README</a></p>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-parsing/metron-parsers-common/message-parser-implementation-notes.html b/current-book/metron-platform/metron-parsing/metron-parsers-common/message-parser-implementation-notes.html
new file mode 100644
index 0000000..081177b
--- /dev/null
+++ b/current-book/metron-platform/metron-parsing/metron-parsers-common/message-parser-implementation-notes.html
@@ -0,0 +1,185 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsing/metron-parsers-common/message-parser-implementation-notes.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – implementation notes</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active "> implementation notes</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-down"></span>Parsing</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/index.html" title="Parsers-common"><span class="icon-chevron-down"></span>Parsers-common</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.html" title="3rdPartyParser"><span class="none"></span>3rdPartyParser</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/ParserChaining.html" title="ParserChaining"><span class="none"></span>ParserChaining</a></li>
+ <li class="active"><a href="#"><span class="none"></span>message-parser-implementation-notes</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/parser-testing.html" title="parser-testing"><span class="none"></span>parser-testing</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/paloalto/index.html" title="Paloalto"><span class="none"></span>Paloalto</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsing-storm/index.html" title="Parsing-storm"><span class="none"></span>Parsing-storm</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1><tt>MessageParser</tt> implementation notes</h1>
+<p><a name="MessageParser_implementation_notes"></a></p>
+<ol style="list-style-type: decimal">
+
+<li>Supporting multiple JSONObject returns from a single byte[] The original <tt>MessageParser</tt> interface supported parsing a message and returning a <tt>List<JSONObject></tt>. Therefore explicitly supporting multiple messages from one input. While this is fine, it only allows for the complete failure of a message for any reason. There can only be one exception thrown. This means that if there <i>are</i> multiple messages in the buffer, any one failure will necessarily fail all of them. To improve on this situation, a new method was added to the <tt>MessageParser</tt> interface (with a default implementation), that introduces a return type to provide not only the JSONObjects produced, but also a <tt>Map</tt> of messages -> throwable.</li>
+</ol>
+<p>To support this in your parser, you should:</p>
+<ul>
+
+<li>Implement the new method</li>
+</ul>
+
+<div>
+<div>
+<pre class="source"> @Override
+ public Optional<MessageParserResult<JSONObject>> parseOptionalResult(byte[] rawMessage)
+</pre></div></div>
+
+<ul>
+
+<li>Implement the original <tt>List<JSONObject> parse(byte[] message)</tt> to delegate to that method such as below:</li>
+</ul>
+
+<div>
+<div>
+<pre class="source"> @Override
+ public List<JSONObject> parse(byte[] rawMessage) {
+ Optional<MessageParserResult<JSONObject>> resultOptional = parseOptionalResult(rawMessage);
+ if (!resultOptional.isPresent()) {
+ return Collections.EMPTY_LIST;
+ }
+ Map<Object,Throwable> errors = resultOptional.get().getMessageThrowables();
+ if (!errors.isEmpty()) {
+ throw new RuntimeException(errors.entrySet().iterator().next().getValue());
+ }
+
+ return resultOptional.get().getMessages();
+ }
+</pre></div></div>
+
+<ul>
+
+<li>
+
+<p>You <i>may</i> want to govern treating the incoming buffer as multiline or not by adding a configuration option for your parser, such as <tt>"multiline":"true"|"false"</tt></p>
+</li>
+<li>
+
+<p>See the org.apache.metron.parsers.GrokParser for an example of this implementation.</p>
+</li>
+</ul>
+<p>The Metron system itself will call the new <tt>parseOptionalResult</tt> method during processing. The default implementation in the interface handles backwards compatability with previous implementations.</p>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-parsing/metron-parsers-common/parser-testing.html b/current-book/metron-platform/metron-parsing/metron-parsers-common/parser-testing.html
new file mode 100644
index 0000000..10be5a9
--- /dev/null
+++ b/current-book/metron-platform/metron-parsing/metron-parsers-common/parser-testing.html
@@ -0,0 +1,161 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsing/metron-parsers-common/parser-testing.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Parser Contribution and Testing</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">Parser Contribution and Testing</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-down"></span>Parsing</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/index.html" title="Parsers-common"><span class="icon-chevron-down"></span>Parsers-common</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.html" title="3rdPartyParser"><span class="none"></span>3rdPartyParser</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/ParserChaining.html" title="ParserChaining"><span class="none"></span>ParserChaining</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/message-parser-implementation-notes.html" title="message-parser-implementation-notes"><span class="none"></span>message-parser-implementation-notes</a></li>
+ <li class="active"><a href="#"><span class="none"></span>parser-testing</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/paloalto/index.html" title="Paloalto"><span class="none"></span>Paloalto</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsing-storm/index.html" title="Parsing-storm"><span class="none"></span>Parsing-storm</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Parser Contribution and Testing</h1>
+<p><a name="Parser_Contribution_and_Testing"></a></p>
+<p>So you want to contribute a parser to Apache Metron. First off, on behalf of the community, thank you very much! Now that you have implemented a parser by writing a java class which implements <tt>org.apache.metron.parsers.interfaces.MessageParser</tt> what are the testing expectations for a new parser?</p>
+<p>It is expected that a new parser have two tests:</p>
+<ul>
+
+<li>A JUnit test directly testing your parser class.</li>
+<li>An Integration test validating that your parser class can parse messages inside the <tt>ParserBolt</tt>.</li>
+</ul>
+<div class="section">
+<h2><a name="The_JUnit_Test"></a>The JUnit Test</h2>
+<p>The JUnit Test should be focused on testing your Parser directly. You should feel free to use mocks or stubs or whatever else you need to completely test that unit of functionality.</p></div>
+<div class="section">
+<h2><a name="The_Integration_Test"></a>The Integration Test</h2>
+<p>Integration tests are more structured. The intent is that the parser that you have implemented can be driven successfully from the appropriate driver, e.g. <tt>org.apache.metron.parsers.bolt.ParserBolt</tt> for Storm.</p>
+<p>To add a new test, just add it to the list of sensorTypes in <tt>org.apache.metron.parsers.integration.ParserIntegrationTest</tt>.</p>
+<p>To setup the tests for a new platform, extend <tt>ParserIntegrationTest</tt>, e.g. as in <tt>org.apache.metron.parsers.integration.StormParserIntegrationTests</tt>. This should be a parameterized test, so that each sensorType gets its own test. Use <tt>StormParserIntegrationTests</tt> as a template for the new platform’s class. The test method should just setup the appropriate <tt>ParserDriver</tt> implementation, and simply call back into the parent to run the test.</p>
+<p>Customized versions of the tests can be added by extending <tt>ParserIntegrationTest</tt> and performing additional setup or validations as needed.</p>
+<p>The way these tests function is by running the <tt>ParserDriver</tt> instance with your specified global configuration and sensor configuration. It will then send your specified sample input data in line-by-line. It will then perform some basic sanity validation:</p>
+<ul>
+
+<li>Ensure no errors were logged</li>
+<li>Execute your specified validation methods</li>
+</ul>
+<div class="section">
+<h3><a name="Validations"></a>Validations</h3>
+<p>Validations are functions which indicate how one should validate the parsed messages. The basic one which is sufficient for most cases is <tt>org.apache.metron.parsers.integration.validation.SampleDataValidation</tt>. This will read the expected results from <tt>metron-integration-test/src/main/sample/data/${sensor_type}/parsed</tt> and validate that the actual parsed message conforms (excluding timestamp).</p>
+<p>If you have special validations required, you may implement your own and return an instance of that in the <tt>getValidations()</tt> method of your Integration Test.</p></div></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/paloalto/index.html b/current-book/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/paloalto/index.html
new file mode 100644
index 0000000..58e48ab
--- /dev/null
+++ b/current-book/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/paloalto/index.html
@@ -0,0 +1,161 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/paloalto/index.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – BasicPaloAltoFirewallParser</title>
+ <link rel="stylesheet" href="../../../../../../../../../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../../../../../../../../../css/site.css" />
+ <link rel="stylesheet" href="../../../../../../../../../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../../../../../../../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../../../../../../../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../../../../../../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">BasicPaloAltoFirewallParser</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../../../../../../../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../../../../../../../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../../../../../../../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../../../../../../../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../../../../../../../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../../../../../../../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../../../../../../../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../../../../../../../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../../../../../../../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-down"></span>Parsing</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/metron-parsers-common/index.html" title="Parsers-common"><span class="icon-chevron-down"></span>Parsers-common</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/metron-parsers-common/3rdPartyParser.html" title="3rdPartyParser"><span class="none"></span>3rdPartyParser</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/metron-parsers-common/ParserChaining.html" title="ParserChaining"><span class="none"></span>ParserChaining</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/metron-parsers-common/message-parser-implementation-notes.html" title="message-parser-implementation-notes"><span class="none"></span>message-parser-implementation-notes</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/metron-parsers-common/parser-testing.html" title="parser-testing"><span class="none"></span>parser-testing</a></li>
+ <li class="active"><a href="#"><span class="none"></span>Paloalto</a></li>
+ </ul>
+</li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/metron-parsing-storm/index.html" title="Parsing-storm"><span class="none"></span>Parsing-storm</a></li>
+ </ul>
+</li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../../../../../../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../../../../../../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../../../../../../../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../../../../../../../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../../../../../../../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../../../../../../../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>BasicPaloAltoFirewallParser</h1>
+<p><a name="BasicPaloAltoFirewallParser"></a></p>
+<div class="section">
+<h2><a name="Introduction"></a>Introduction</h2>
+<p>The parser is able to parse logs generated by Palo Alto firewall devices powered by Pan OS. The supported log format is CSV.</p>
+<p>The supported log types and versions</p>
+<table border="0" class="table table-striped">
+<thead>
+
+<tr class="a">
+<th> Log type </th>
+<th> Pan OS version </th></tr>
+</thead><tbody>
+
+<tr class="b">
+<td> Traffic </td>
+<td> 6.0, 6.1, 7.0, 8.0 </td></tr>
+<tr class="a">
+<td> Threat </td>
+<td> 6.0, 6.1, 7.0, 8.0 </td></tr>
+<tr class="b">
+<td> Config </td>
+<td> 6.1, 7.0, 8.0 </td></tr>
+<tr class="a">
+<td> System </td>
+<td> 6.1, 7.0, 8.0 </td></tr>
+</tbody>
+</table></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-parsing/metron-parsers/index.html b/current-book/metron-platform/metron-parsing/metron-parsers/index.html
new file mode 100644
index 0000000..88b24cb
--- /dev/null
+++ b/current-book/metron-platform/metron-parsing/metron-parsers/index.html
@@ -0,0 +1,148 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsing/metron-parsers/index.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – metron-parsers</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">metron-parsers</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-down"></span>Parsing</a>
+ <ul class="nav nav-list">
+ <li class="active"><a href="#"><span class="icon-chevron-down"></span>Parsers</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/index.html" title="Paloalto"><span class="none"></span>Paloalto</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/index.html" title="Parsers-common"><span class="icon-chevron-right"></span>Parsers-common</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsing-storm/index.html" title="Parsing-storm"><span class="none"></span>Parsing-storm</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>metron-parsers</h1>
+<p><a name="metron-parsers"></a></p>
+<p>metron-parsers contains the included parsers that aren’t our basic parsers.</p>
+<p>The included parsers are:</p>
+<ul>
+
+<li>Asa</li>
+<li>CEF</li>
+<li>FireEye</li>
+<li>ISE</li>
+<li>Lancope</li>
+<li>Logstash</li>
+<li>PaloAlto</li>
+<li>Snort</li>
+<li>Sourcefire</li>
+<li>Websphere</li>
+</ul>
+<p>The basic parsers and their details can be found at <a href="../metron-parsers-common/index.html#README.md">README</a>.</p>
+<p>More details on these parsers and the overall architecture can be found in the metron-parsing <a href="..#README.md">README</a>.</p>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/index.html b/current-book/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/index.html
new file mode 100644
index 0000000..ac39701
--- /dev/null
+++ b/current-book/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/index.html
@@ -0,0 +1,157 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/index.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – BasicPaloAltoFirewallParser</title>
+ <link rel="stylesheet" href="../../../../../../../../../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../../../../../../../../../css/site.css" />
+ <link rel="stylesheet" href="../../../../../../../../../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../../../../../../../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../../../../../../../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../../../../../../../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">BasicPaloAltoFirewallParser</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../../../../../../../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../../../../../../../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../../../../../../../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../../../../../../../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../../../../../../../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../../../../../../../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../../../../../../../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../../../../../../../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../../../../../../../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-down"></span>Parsing</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-down"></span>Parsers</a>
+ <ul class="nav nav-list">
+ <li class="active"><a href="#"><span class="none"></span>Paloalto</a></li>
+ </ul>
+</li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/metron-parsers-common/index.html" title="Parsers-common"><span class="icon-chevron-right"></span>Parsers-common</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-parsing/metron-parsing-storm/index.html" title="Parsing-storm"><span class="none"></span>Parsing-storm</a></li>
+ </ul>
+</li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../../../../../../../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../../../../../../../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../../../../../../../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../../../../../../../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../../../../../../../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../../../../../../../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../../../../../../../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>BasicPaloAltoFirewallParser</h1>
+<p><a name="BasicPaloAltoFirewallParser"></a></p>
+<div class="section">
+<h2><a name="Introduction"></a>Introduction</h2>
+<p>The parser is able to parse logs generated by Palo Alto firewall devices powered by Pan OS. The supported log format is CSV.</p>
+<p>The supported log types and versions</p>
+<table border="0" class="table table-striped">
+<thead>
+
+<tr class="a">
+<th> Log type </th>
+<th> Pan OS version </th></tr>
+</thead><tbody>
+
+<tr class="b">
+<td> Traffic </td>
+<td> 6.0, 6.1, 7.0, 8.0 </td></tr>
+<tr class="a">
+<td> Threat </td>
+<td> 6.0, 6.1, 7.0, 8.0 </td></tr>
+<tr class="b">
+<td> Config </td>
+<td> 6.1, 7.0, 8.0 </td></tr>
+<tr class="a">
+<td> System </td>
+<td> 6.1, 7.0, 8.0 </td></tr>
+</tbody>
+</table></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-parsing/metron-parsing-storm/index.html b/current-book/metron-platform/metron-parsing/metron-parsing-storm/index.html
new file mode 100644
index 0000000..dd6c644
--- /dev/null
+++ b/current-book/metron-platform/metron-parsing/metron-parsing-storm/index.html
@@ -0,0 +1,278 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsing/metron-parsing-storm/index.md at 2019-05-14
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Metron – Parsers</title>
+ <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.7.min.css" />
+ <link rel="stylesheet" href="../../../css/site.css" />
+ <link rel="stylesheet" href="../../../css/print.css" media="print" />
+ <script type="text/javascript" src="../../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+ $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+ </script>
+ </head>
+ <body class="topBarDisabled">
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div>
+ <div class="pull-right"></div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+ <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+ <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+ <li class=""><a href="../../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+ <li class="active ">Parsers</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
+ </ul>
+ </div>
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header">User Documentation</li>
+ <li><a href="../../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+ <li><a href="../../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+ <li><a href="../../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+ <li><a href="../../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+ <li><a href="../../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+ <li><a href="../../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+ <li><a href="../../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
+ <li><a href="../../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+ <li><a href="../../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+ <li><a href="../../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+ <li><a href="../../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+ <li><a href="../../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
+ <li><a href="../../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+ <li><a href="../../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+ <li><a href="../../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-down"></span>Parsing</a>
+ <ul class="nav nav-list">
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../../metron-platform/metron-parsing/metron-parsers-common/index.html" title="Parsers-common"><span class="icon-chevron-right"></span>Parsers-common</a></li>
+ <li class="active"><a href="#"><span class="none"></span>Parsing-storm</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+ <li><a href="../../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
+ <li><a href="../../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+ </ul>
+</li>
+ <li><a href="../../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+ <li><a href="../../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+ <li><a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+ <li><a href="../../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+ <li><a href="../../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+ </ul>
+</li>
+</ul>
+ <hr />
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" /></a>
+ </div>
+ </div>
+ </div>
+ <div id="bodyColumn" class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Parsers</h1>
+<p><a name="Parsers"></a></p>
+<div class="section">
+<h2><a name="Introduction"></a>Introduction</h2>
+<p>Metron’s parsers can be run in Storm topologies, complete with their own set of configuration options (e.g. parallelism). A script is provided to deploy a parser as a Storm topologoy.</p></div>
+<div class="section">
+<h2><a name="Parser_Configuration"></a>Parser Configuration</h2>
+<ul>
+
+<li><tt>spoutParallelism</tt> : The kafka spout parallelism (default to <tt>1</tt>). This can be overridden on the command line, and if there are multiple sensors should be in a comma separated list in the same order as the sensors.</li>
+<li><tt>spoutNumTasks</tt> : The number of tasks for the spout (default to <tt>1</tt>). This can be overridden on the command line, and if there are multiple sensors should be in a comma separated list in the same order as the sensors.</li>
+<li><tt>parserParallelism</tt> : The parser bolt parallelism (default to <tt>1</tt>). If there are multiple sensors, the last one’s configuration will be used. This can be overridden on the command line.</li>
+<li><tt>parserNumTasks</tt> : The number of tasks for the parser bolt (default to <tt>1</tt>). If there are multiple sensors, the last one’s configuration will be used. This can be overridden on the command line.</li>
+<li><tt>errorWriterParallelism</tt> : The error writer bolt parallelism (default to <tt>1</tt>). This can be overridden on the command line.</li>
+<li><tt>errorWriterNumTasks</tt> : The number of tasks for the error writer bolt (default to <tt>1</tt>). This can be overridden on the command line.</li>
+<li><tt>numWorkers</tt> : The number of workers to use in the topology (default is the storm default of <tt>1</tt>).</li>
+<li><tt>numAckers</tt> : The number of acker executors to use in the topology (default is the storm default of <tt>1</tt>).</li>
+<li><tt>spoutConfig</tt> : A map representing a custom spout config (this is a map). If there are multiple sensors, the configs will be merged with the last specified taking precedence. This can be overridden on the command line.</li>
+<li><tt>stormConfig</tt> : The storm config to use (this is a map). This can be overridden on the command line. If both are specified, they are merged with CLI properties taking precedence.</li>
+</ul>
+<p><a name="Starting_the_Parser_Topology"></a></p>
+<h1>Starting the Parser Topology</h1>
+<p>Starting a particular parser topology on a running Metron deployment is as easy as running the <tt>start_parser_topology.sh</tt> script located in <tt>$METRON_HOME/bin</tt>. This utility will allow you to configure and start the running topology assuming that the sensor specific parser configuration exists within zookeeper.</p>
+<p>The usage for <tt>start_parser_topology.sh</tt> is as follows:</p>
+
+<div>
+<div>
+<pre class="source">usage: start_parser_topology.sh
+ -e,--extra_topology_options <JSON_FILE> Extra options in the form
+ of a JSON file with a map
+ for content.
+ -esc,--extra_kafka_spout_config <JSON_FILE> Extra spout config options
+ in the form of a JSON file
+ with a map for content.
+ Possible keys are:
+ retryDelayMaxMs,retryDelay
+ Multiplier,retryInitialDel
+ ayMs,stateUpdateIntervalMs
+ ,bufferSizeBytes,fetchMaxW
+ ait,fetchSizeBytes,maxOffs
+ etBehind,metricsTimeBucket
+ SizeInSecs,socketTimeoutMs
+ -ewnt,--error_writer_num_tasks <NUM_TASKS> Error Writer Num Tasks
+ -ewp,--error_writer_p <PARALLELISM_HINT> Error Writer Parallelism
+ Hint
+ -h,--help This screen
+ -iwnt,--invalid_writer_num_tasks <NUM_TASKS> Invalid Writer Num Tasks
+ -iwp,--invalid_writer_p <PARALLELISM_HINT> Invalid Message Writer Parallelism Hint
+ -k,--kafka <BROKER_URL> Kafka Broker URL
+ -ksp,--kafka_security_protocol <SECURITY_PROTOCOL> Kafka Security Protocol
+ -mt,--message_timeout <TIMEOUT_IN_SECS> Message Timeout in Seconds
+ -mtp,--max_task_parallelism <MAX_TASK> Max task parallelism
+ -na,--num_ackers <NUM_ACKERS> Number of Ackers
+ -nw,--num_workers <NUM_WORKERS> Number of Workers
+ -ot,--output_topic <KAFKA_TOPIC> Output Kafka Topic
+ -pnt,--parser_num_tasks <NUM_TASKS> Parser Num Tasks
+ -pp,--parser_p <PARALLELISM_HINT> Parser Parallelism Hint
+ -s,--sensor <SENSOR_TYPE> Sensor Type
+ -snt,--spout_num_tasks <NUM_TASKS> Spout Num Tasks
+ -sp,--spout_p <SPOUT_PARALLELISM_HINT> Spout Parallelism Hint
+ -t,--test <TEST> Run in Test Mode
+ -z,--zk <ZK_QUORUM> Zookeeper Quroum URL
+ (zk1:2181,zk2:2181,...
+</pre></div></div>
+</div>
+<div class="section">
+<h2><a name="The_--extra_kafka_spout_config_Option"></a>The <tt>--extra_kafka_spout_config</tt> Option</h2>
+<p>These options are intended to configure the Storm Kafka Spout more completely. These options can be specified in a JSON file containing a map associating the kafka spout configuration parameter to a value. The range of values possible to configure are:</p>
+<ul>
+
+<li><tt>spout.pollTimeoutMs</tt> - Specifies the time, in milliseconds, spent waiting in poll if data is not available. Default is 2s</li>
+<li><tt>spout.firstPollOffsetStrategy</tt> - Sets the offset used by the Kafka spout in the first poll to Kafka broker upon process start. One of
+<ul>
+
+<li><tt>EARLIEST</tt></li>
+<li><tt>LATEST</tt></li>
+<li><tt>UNCOMMITTED_EARLIEST</tt> - Last uncommitted and if offsets aren’t found, defaults to earliest. NOTE: This is the default.</li>
+<li><tt>UNCOMMITTED_LATEST</tt> - Last uncommitted and if offsets aren’t found, defaults to latest.</li>
+</ul>
+</li>
+<li><tt>spout.offsetCommitPeriodMs</tt> - Specifies the period, in milliseconds, the offset commit task is periodically called. Default is 15s.</li>
+<li><tt>spout.maxUncommittedOffsets</tt> - Defines the max number of polled offsets (records) that can be pending commit, before another poll can take place. Once this limit is reached, no more offsets (records) can be polled until the next successful commit(s) sets the number of pending offsets bellow the threshold. The default is 10,000,000.</li>
+<li><tt>spout.maxRetries</tt> - Defines the max number of retrials in case of tuple failure. The default is to retry forever, which means that no new records are committed until the previous polled records have been acked. This guarantees at once delivery of all the previously polled records. By specifying a finite value for maxRetries, the user decides to sacrifice guarantee of delivery for the previous polled records in favor of processing more records.</li>
+<li>Any of the configs in the Consumer API for <a class="externalLink" href="http://kafka.apache.org/0100/documentation.html#newconsumerconfigs">Kafka 0.10.x</a></li>
+</ul>
+<p>For instance, creating a JSON file which will set the offsets to <tt>UNCOMMITTED_EARLIEST</tt></p>
+
+<div>
+<div>
+<pre class="source">{
+ "spout.firstPollOffsetStrategy" : "UNCOMMITTED_EARLIEST"
+}
+</pre></div></div>
+
+<p>This would be loaded by passing the file as argument to <tt>--extra_kafka_spout_config</tt></p></div>
+<div class="section">
+<h2><a name="The_--extra_topology_options_Option"></a>The <tt>--extra_topology_options</tt> Option</h2>
+<p>These options are intended to be Storm configuration options and will live in a JSON file which will be loaded into the Storm config. For instance, if you wanted to set a storm property on the config called <tt>topology.ticks.tuple.freq.secs</tt> to 1000 and <tt>storm.local.dir</tt> to <tt>/opt/my/path</tt> you could create a file called <tt>custom_config.json</tt> containing</p>
+
+<div>
+<div>
+<pre class="source">{
+ "topology.ticks.tuple.freq.secs" : 1000,
+ "storm.local.dir" : "/opt/my/path"
+}
+</pre></div></div>
+
+<p>and pass <tt>--extra_topology_options custom_config.json</tt> to <tt>start_parser_topology.sh</tt>.</p></div>
+<div class="section">
+<h2><a name="Parser_Topology"></a>Parser Topology</h2>
+<p>The enrichment topology as started by the <tt>$METRON_HOME/bin/start_parser_topology.sh</tt> script uses a default of one executor per bolt. In a real production system, this should be customized by modifying the arguments sent to this utility.</p>
+<ul>
+
+<li>Topology Wide
+<ul>
+
+<li><tt>--num_workers</tt> : The number of workers for the topology</li>
+<li><tt>--num_ackers</tt> : The number of ackers for the topology</li>
+</ul>
+</li>
+<li>The Kafka Spout
+<ul>
+
+<li><tt>--spout_num_tasks</tt> : The number of tasks for the spout</li>
+<li><tt>--spout_p</tt> : The parallelism hint for the spout</li>
+<li>Ensure that the spout has enough parallelism so that it can dedicate a worker per partition in your kafka topic.</li>
+</ul>
+</li>
+<li>The Parser Bolt
+<ul>
+
+<li><tt>--parser_num_tasks</tt> : The number of tasks for the parser bolt</li>
+<li><tt>--parser_p</tt> : The parallelism hint for the spout</li>
+<li>This is bolt that gets the most processing, so ensure that it is configured with sufficient parallelism to match your throughput expectations.</li>
+</ul>
+</li>
+<li>The Error Message Writer Bolt
+<ul>
+
+<li><tt>--error_writer_num_tasks</tt> : The number of tasks for the error writer bolt</li>
+<li><tt>--error_writer_p</tt> : The parallelism hint for the error writer bolt</li>
+</ul>
+</li>
+</ul>
+<p>Finally, if workers and executors are new to you, the following might be of use to you:</p>
+<ul>
+
+<li><a class="externalLink" href="http://www.michael-noll.com/blog/2012/10/16/understanding-the-parallelism-of-a-storm-topology/">Understanding the Parallelism of a Storm Topology</a></li>
+</ul></div>
+<div class="section">
+<h2><a name="Parser_Aggregation"></a>Parser Aggregation</h2>
+<p>For performance reasons, multiple sensors can be aggregated into a single Storm topology. When this is done, there will be multiple Kafka spouts, but only a single parser bolt which will handle delegating to the correct parser as needed. There are some constraints around this, in particular regarding some configuration. Additionally, all sensors must flow to the same error topic. The Kafka topic is retrieved from the input Tuple itself.</p>
+<p>A worked example of this can be found in the <a href="../../../use-cases/parser_chaining/index.html#aggregated-parsers-with-parser-chaining">Parser Chaining use case</a>.</p></div>
+ </div>
+ </div>
+ </div>
+ <hr/>
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+ and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+ </div>
+ </div>
+ </footer>
+ </body>
+</html>
diff --git a/current-book/metron-platform/metron-pcap-backend/index.html b/current-book/metron-platform/metron-pcap-backend/index.html
index 9c3278a..4223a8e 100644
--- a/current-book/metron-platform/metron-pcap-backend/index.html
+++ b/current-book/metron-platform/metron-pcap-backend/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-pcap-backend/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-pcap-backend/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron PCAP Backend</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron PCAP Backend</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -59,10 +57,11 @@
<li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
<li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
<li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
<li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
<li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
- <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li class="active"><a href="#"><span class="none"></span>Pcap-backend</a></li>
<li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
<li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
diff --git a/current-book/metron-platform/metron-solr/index.html b/current-book/metron-platform/metron-solr/index.html
index f0d8362..0f787a5 100644
--- a/current-book/metron-platform/metron-solr/index.html
+++ b/current-book/metron-platform/metron-solr/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-solr/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-solr/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Solr in Metron</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Solr in Metron</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -59,10 +57,11 @@
<li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
<li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
<li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
<li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
<li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
- <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
<li class="active"><a href="#"><span class="none"></span>Solr</a></li>
<li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
@@ -210,8 +209,8 @@
<li>Stop the Metron Indexing component in Ambari.</li>
<li>Update Ambari UI -> Services -> Metron -> Configs -> Index Settings -> Solr Zookeeper Urls to match the Solr installation described in the previous section.</li>
<li>Change Ambari UI -> Services -> Metron -> Configs -> Indexing -> Index Writer - Random Access -> Random Access Search Engine to <tt>Solr</tt>.</li>
-<li>Set the <tt>source.type.field</tt> property to <tt>source.type</tt> in the <a href="../metron-common/index.html#Global_Configuration">Global Configuration</a>.</li>
-<li>Set the <tt>threat.triage.score.field</tt> property to <tt>threat.triage.score</tt> in the <a href="../metron-common/index.html#Global_Configuration">Global Configuration</a>.</li>
+<li>Change Ambari UI -> Services -> Metron -> Configs -> REST -> Source Type Field Name to <tt>source.type</tt>.</li>
+<li>Change Ambari UI -> Services -> Metron -> Configs -> REST -> Threat Triage Score Field Name to <tt>threat.triage.score</tt>.</li>
<li>Start the Metron Indexing component in Ambari.</li>
<li>Restart Metron REST and the Alerts UI in Ambari.</li>
</ol>
diff --git a/current-book/metron-platform/metron-writer/index.html b/current-book/metron-platform/metron-writer/index.html
index 1469d6f..3e9700b 100644
--- a/current-book/metron-platform/metron-writer/index.html
+++ b/current-book/metron-platform/metron-writer/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-writer/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-writer/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Writer</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Writer</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
<ul class="nav nav-list">
<li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
@@ -59,10 +57,11 @@
<li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
<li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
<li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+ <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li>
<li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
<li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
<li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
- <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+ <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li>
<li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
<li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
<li class="active"><a href="#"><span class="none"></span>Writer</a></li>
@@ -109,13 +108,113 @@
<h2><a name="Introduction"></a>Introduction</h2>
<p>The writer module provides some utilties for writing to outside components from within Storm. This includes managing bulk writing. An implemention is included for writing to HDFS in this module. Other writers can be found in their own modules.</p></div>
<div class="section">
+<h2><a name="Bulk_Message_Writing"></a>Bulk Message Writing</h2>
+<p>Most external components encourage messages to be written in batches for performance reasons. The writer module includes an abstraction for doing this in an efficient manner. This abstraction provides the following features:</p>
+<ul>
+
+<li>A high-level <tt>BulkWriterComponent</tt> class that manages a per-sensor cache of batched messages and flushes when appropriate</li>
+<li>An extension point for determining when a batch should be flushed</li>
+<li>An extention point for handling a bulk message write response after a batch has been flushed</li>
+</ul>
+<div class="section">
+<h3><a name="Flush_Policies"></a>Flush Policies</h3>
+<p>Flushing behavior is controlled by a collection of <tt>FlushPolicy</tt> objects. They are responsible for 2 things:</p>
+<ol style="list-style-type: decimal">
+
+<li>Determining when a batch should be flushed</li>
+<li>Handling a <tt>BulkWriterResponse</tt> after a batch is flushed and messages are written with a <tt>BulkMessageWriter</tt></li>
+</ol>
+<p>The <tt>FlushPolicy</tt> interface defines methods for handling these responsiblities:</p>
+<ul>
+
+<li><tt>boolean shouldFlush(String sensorType, WriterConfiguration configurations, List<BulkMessage<MESSAGE_T>> messages)</tt></li>
+<li><tt>void onFlush(String sensorType, BulkWriterResponse response)</tt></li>
+</ul>
+<p>There are 2 <tt>FlushPolicy</tt> implementations included by default:</p>
+<ul>
+
+<li>The <tt>BatchSizePolicy</tt> will flush a batch whenever the batch size reaches a configured value. This configuration value is represented by the <tt>batchSize</tt> property in either the parser, enrichment or indexing configuration (whichever is appropriate in the current context).</li>
+<li>The <tt>BatchTimeoutPolicy</tt> will flush a batch whenever the batch timeout has elapsed. This configuration value is represented by the <tt>batchTimeout</tt> property in either the parser, enrichment or indexing configuration (whichever is appropriate in the current context). A <tt>maxBatchTimeout</tt> is set at creation time and serves as the ceiling for a batch timeout. In Storm topologies, this value is set to 1/2 the tuple timeout setting to ensure messages are always flushed before their tuples timeout. After a batch is flushed, the batch timer is reset for that sensor type.</li>
+</ul>
+<p>For example, a configuration that sets the <tt>batchSize</tt> and <tt>batchTimeout</tt> in a parser topology will look like:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "parserClassName": "org.apache.metron.parsers.bro.BasicBroParser",
+ "sensorTopic": "bro",
+ "parserConfig": {
+ "batchSize": 5
+ "batchTimeout": 2
+ }
+}
+</pre></div></div>
+
+<p>Similarly for the enrichment topology (configured in the <a href="../metron-common/index.html#Global_Configuration">Global Configuration</a>):</p>
+
+<div>
+<div>
+<pre class="source">{
+ "enrichment.writer.batchSize": "5",
+ "enrichment.writer.batchTimeout": "2",
+ ...
+}
+</pre></div></div>
+
+<p>And finally for the indexing topology:</p>
+
+<div>
+<div>
+<pre class="source">{
+ "elasticsearch": {
+ "index": "bro",
+ "batchSize": 5,
+ "batchTimeout": 2,
+ "enabled": true
+ },
+ ...
+}
+</pre></div></div>
+
+<p>Additional policies can be added as needed. For example, an <tt>AckTuplesPolicy</tt> is added in the Storm bolts to handle acking tuples after a batch is flushed.</p></div>
+<div class="section">
+<h3><a name="Bulk_Writing_Workflow"></a>Bulk Writing Workflow</h3>
+<p>The <tt>BulkWriterComponent</tt> class collects messages in separate sensor-specific caches. This class is instantiated and supplied to classes that need to write messages to external components. A collection of default <tt>FlushPolicy</tt> implementations are created by default with the option of passing in additional <tt>FlushPolicy</tt> objects as needed.</p>
+<p>Batching and writing messages follows this process:</p>
+<ol style="list-style-type: decimal">
+
+<li>A single message is passed to the <tt>BulkWriterComponent.write</tt> method and stored in the appropriate cache based on the sensor type. A <tt>BulkMessageWriter</tt> is also supplied to do the actual writing when messages are flushed.</li>
+<li>The collection of <tt>FlushPolicy</tt> implementations are checked and a batch is flushed whenever the first <tt>FlushPolicy.shouldFlush</tt> returns true.</li>
+<li>If no policies signal a flush, then nothing happens. If a policy does signal a flush, the batch of messages for that sensor are written with the supplied <tt>BulkMessageWriter</tt>. Each <tt>FlushPolicy.onFlush</tt> method is then called with the <tt>BulkWriterResponse</tt>.</li>
+<li>If a sensor type has been disabled, it’s batch is flushed immediately (<tt>FlushPolicy.shouldFlush</tt> is not checked).</li>
+<li>A <tt>BulkWriterComponent.flushAll</tt> method is available that immediately calls the <tt>FlushPolicy.shouldFlush</tt> methods for each sensor type in the cache. This should be called periodically by the class containing <tt>BulkWriterComponent</tt> to ensure messages are not left sitting in the cache. For example, the Storm bolts call this whenever a tick tuple is received.</li>
+</ol></div>
+<div class="section">
+<h3><a name="Logging"></a>Logging</h3>
+<p>Logging can be enabled for the classes described in this section to provide insight into how messages are being batched and flushed. This can be an important tool when performance tuning. Setting the log level to <tt>DEBUG</tt> on the <tt>org.apache.metron.writer</tt> package will produce detailed information about when batches are flushed, which sensor a flushed batch corresponds to, which policy caused the flush, and how long it took to write the batch.</p></div>
+<div class="section">
+<h3><a name="Performance_Tuning"></a>Performance Tuning</h3>
+<p>The primary purpose of the Bulk Message Writing abstraction is to enable efficient writing to external components. This section provides recommendations and strategies for doing that. It is assumed that most Metron installations will include a variety of sensors with different volumes and velocities. Different sensors will likely need to be tuned differently.</p>
+<div class="section">
+<h4><a name="Set_batches_sizes_higher_for_high_volume_sensors"></a>Set batches sizes higher for high volume sensors</h4>
+<p>High volume sensors should be identified and configured with higher batch sizes (1000+ is recommended). Use logging to verify these batches are in fact filling up. The number of actual message written should match the batch size. Keep in mind that large batch sizes also require more memory to hold messages. Streaming engines like Storm limit how many messages can be processed at a time (the <tt>topology.max.spout.pending</tt> setting).</p></div>
+<div class="section">
+<h4><a name="Set_batch_timeouts_lower_for_low_volume_sensors"></a>Set batch timeouts lower for low volume sensors</h4>
+<p>Low volume sensors make take longer to fill up a batch, especially if the batch size is set higher. This can be undesirable because messages may stay cached for longer than necessary, consuming memory and increasing latency for that sensor type.</p></div>
+<div class="section">
+<h4><a name="Be_careful_with_threads"></a>Be careful with threads</h4>
+<p>Each thread (executor in Storm) maintains it’s own message cache. Allocating too many threads will cause messages to be spread too thin across separate caches and batches won’t fill up completely. This should be balanced with having enough threads to take advantage of any parallel write capability offered by the endpoint that’s being written to.</p></div>
+<div class="section">
+<h4><a name="Watch_for_high_write_times"></a>Watch for high write times</h4>
+<p>Use logging to evaluate write timing. Unusually high write times can indicate that an endpoint is not configured correctly or undersized.</p></div></div></div>
+<div class="section">
<h2><a name="Kafka_Writer"></a>Kafka Writer</h2>
<p>We have an implementation of a writer which will write batches of messages to Kafka. An interesting aspect of this writer is that it can be configured to allow users to specify a message field which contains the topic for the message.</p>
<p>The configuration for this writer is held in the individual Sensor Configurations:</p>
<ul>
<li><a href="../metron-enrichment/index.html#sensor-enrichment-configuration">Enrichment</a> under the <tt>config</tt> element</li>
-<li><a href="../metron-parsers/index.html#parser-configuration">Parsers</a> in the <tt>parserConfig</tt> element</li>
+<li><a href="../metron-parsers-common/index.html#parser-configuration">Parsers</a> in the <tt>parserConfig</tt> element</li>
<li>Profiler - Unsupported currently</li>
</ul>
<p>In each of these, the kafka writer can be configured via a map which has the following elements:</p>
@@ -152,6 +251,7 @@
</pre></div></div>
<p>This means that all output will land in <tt>/apps/metron/</tt>. With no further adjustment, it will be <tt>/apps/metron/<sensor>/</tt>. However, by modifying the sensor’s JSON config, it is possible to provide additional pathing based on the the message itself.</p>
+<p>The output format of a file will be <tt>{prefix}{componentId}-{taskId}-{rotationNum}-{timestamp}{extension}</tt>. Notably, because of the way file rotations are handled by the HdfsWriter, <tt>rotationNum</tt> will always be 0, but RotationActions still get executed normally.</p>
<p>E.g.</p>
<div>
diff --git a/current-book/metron-sensors/fastcapa/index.html b/current-book/metron-sensors/fastcapa/index.html
index 73f176d..0250461 100644
--- a/current-book/metron-sensors/fastcapa/index.html
+++ b/current-book/metron-sensors/fastcapa/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/fastcapa/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/fastcapa/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Fastcapa</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Fastcapa</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-down"></span>Sensors</a>
<ul class="nav nav-list">
diff --git a/current-book/metron-sensors/index.html b/current-book/metron-sensors/index.html
index b28fe48..22f12d9 100644
--- a/current-book/metron-sensors/index.html
+++ b/current-book/metron-sensors/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Metron Sensors</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Metron Sensors</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li class="active"><a href="#"><span class="icon-chevron-down"></span>Sensors</a>
<ul class="nav nav-list">
diff --git a/current-book/metron-sensors/pycapa/index.html b/current-book/metron-sensors/pycapa/index.html
index e0a3045..84ec3d5 100644
--- a/current-book/metron-sensors/pycapa/index.html
+++ b/current-book/metron-sensors/pycapa/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/pycapa/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/pycapa/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Pycapa</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Pycapa</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-down"></span>Sensors</a>
<ul class="nav nav-list">
diff --git a/current-book/metron-stellar/stellar-3rd-party-example/index.html b/current-book/metron-stellar/stellar-3rd-party-example/index.html
index 34501bc..31b92d7 100644
--- a/current-book/metron-stellar/stellar-3rd-party-example/index.html
+++ b/current-book/metron-stellar/stellar-3rd-party-example/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-3rd-party-example/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-3rd-party-example/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Introduction</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Introduction</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li class="active"><a href="#"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-stellar/stellar-common/3rdPartyStellar.html b/current-book/metron-stellar/stellar-common/3rdPartyStellar.html
index bcee9dc..10e080f 100644
--- a/current-book/metron-stellar/stellar-common/3rdPartyStellar.html
+++ b/current-book/metron-stellar/stellar-common/3rdPartyStellar.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-common/3rdPartyStellar.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-common/3rdPartyStellar.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Custom Stellar Functions</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Custom Stellar Functions</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/metron-stellar/stellar-common/index.html b/current-book/metron-stellar/stellar-common/index.html
index ee93071..1f3b28b 100644
--- a/current-book/metron-stellar/stellar-common/index.html
+++ b/current-book/metron-stellar/stellar-common/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-common/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-common/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Stellar Language</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Stellar Language</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -333,171 +331,179 @@
<tr class="a">
<td> <a href="#APPEND_IF_MISSING"> <tt>APPEND_IF_MISSING</tt></a> </td></tr>
<tr class="b">
+<td> <a href="#ASN_GET"> <tt>ASN_GET</tt></a> </td></tr>
+<tr class="a">
<td> <a href="../../metron-analytics/metron-statistics/index.html#BIN"> <tt>BIN</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#BLOOM_ADD"> <tt>BLOOM_ADD</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#BLOOM_EXISTS"> <tt>BLOOM_EXISTS</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#BLOOM_INIT"> <tt>BLOOM_INIT</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#BLOOM_MERGE"> <tt>BLOOM_MERGE</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#CEILING"> <tt>CEILING</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#COS"> <tt>COS</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#CHOP"> <tt>CHOP</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#CHOMP"> <tt>CHOMP</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#COUNT_MATCHES"> <tt>COUNT_MATCHES</tt></a> </td></tr>
-<tr class="b">
-<td> <a href="#DATE_FORMAT"> <tt>DATE_FORMAT</tt></a></td></tr>
<tr class="a">
+<td> <a href="#DATE_FORMAT"> <tt>DATE_FORMAT</tt></a> </td></tr>
+<tr class="b">
<td> <a href="#DAY_OF_MONTH"> <tt>DAY_OF_MONTH</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#DAY_OF_WEEK"> <tt>DAY_OF_WEEK</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#DAY_OF_YEAR"> <tt>DAY_OF_YEAR</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#DECODE"> <tt>DECODE</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#DOMAIN_REMOVE_SUBDOMAINS"> <tt>DOMAIN_REMOVE_SUBDOMAINS</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#DOMAIN_REMOVE_TLD"> <tt>DOMAIN_REMOVE_TLD</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#DOMAIN_TO_TLD"> <tt>DOMAIN_TO_TLD</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#ENCODE"> <tt>ENCODE</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#ENDS_WITH"> <tt>ENDS_WITH</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#ENRICHMENT_EXISTS"> <tt>ENRICHMENT_EXISTS</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#ENRICHMENT_GET"> <tt>ENRICHMENT_GET</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#EXP"> <tt>EXP</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#FILL_LEFT"> <tt>FILL_LEFT</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#FILL_RIGHT"> <tt>FILL_RIGHT</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#FILTER"> <tt>FILTER</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#FLOOR"> <tt>FLOOR</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#FORMAT"> <tt>FORMAT</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#FUZZY_LANGS"> <tt>FUZZY_LANGS</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#FUZZY_SCORE"> <tt>FUZZY_SCORE</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#GEO_GET"> <tt>GEO_GET</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#GEOHASH_CENTROID"> <tt>GEOHASH_CENTROID</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#GEOHASH_DIST"> <tt>GEOHASH_DIST</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#GEOHASH_FROM_LATLONG"> <tt>GEOHASH_FROM_LATLONG</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#GEOHASH_FROM_LOC"> <tt>GEOHASH_FROM_LOC</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#GEOHASH_MAX_DIST"> <tt>GEOHASH_MAX_DIST</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#GEOHASH_TO_LATLONG"> <tt>GEOHASH_TO_LATLONG</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#GET"> <tt>GET</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#GET_FIRST"> <tt>GET_FIRST</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#GET_LAST"> <tt>GET_LAST</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#GET_SUPPORTED_ENCODINGS"> <tt>GET_SUPPORTED_ENCODINGS</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#HASH"> <tt>HASH</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="../../metron-analytics/metron-statistics/index.html#HLLP_ADD"> <tt>HLLP_ADD</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="../../metron-analytics/metron-statistics/index.html#HLLP_CARDINALITY"> <tt>HLLP_CARDINALITY</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="../../metron-analytics/metron-statistics/index.html#HLLP_INIT"> <tt>HLLP_INIT</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="../../metron-analytics/metron-statistics/index.html#HLLP_MERGE"> <tt>HLLP_MERGE</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#IN_SUBNET"> <tt>IN_SUBNET</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="../../metron-analytics/metron-statistics/index.html#IT_ENTROPY"> <tt>IT_ENTROPY</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#IS_DATE"> <tt>IS_DATE</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#IS_ENCODING"> <tt>IS_ENCODING</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#IS_DOMAIN"> <tt>IS_DOMAIN</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#IS_EMAIL"> <tt>IS_EMAIL</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#IS_EMPTY"> <tt>IS_EMPTY</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#IS_INTEGER"> <tt>IS_INTEGER</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#IS_IP"> <tt>IS_IP</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#IS_NAN"> <tt>IS_NAN</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#IS_URL"> <tt>IS_URL</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#JOIN"> <tt>JOIN</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#KAFKA_GET"> <tt>KAFKA_GET</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#KAFKA_PROPS"> <tt>KAFKA_PROPS</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#KAFKA_PUT"> <tt>KAFKA_PUT</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#KAFKA_TAIL"> <tt>KAFKA_TAIL</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#LENGTH"> <tt>LENGTH</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#LIST_ADD"> <tt>LIST_ADD</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#LOG2"> <tt>LOG2</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#LOG10"> <tt>LOG10</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#LN"> <tt>LN</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#MAAS_GET_ENDPOINT"> <tt>MAAS_GET_ENDPOINT</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#MAAS_MODEL_APPLY"> <tt>MAAS_MODEL_APPLY</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#MAP"> <tt>MAP</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#MAP_EXISTS"> <tt>MAP_EXISTS</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#MAP_GET"> <tt>MAP_GET</tt></a> </td></tr>
+<tr class="a">
+<td> <a href="#MAP_MERGE"> <tt>MAP_MERGE</tt></a> </td></tr>
<tr class="b">
+<td> <a href="#MAP_PUT"> <tt>MAP_PUT</tt></a> </td></tr>
+<tr class="a">
<td> <a href="#MAX"> <tt>MAX</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#MIN"> <tt>MIN</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#MONTH"> <tt>MONTH</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#MULTISET_ADD"> <tt>MULTISET_ADD</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#MULTISET_INIT"> <tt>MULTISET_INIT</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#MULTISET_MERGE"> <tt>MULTISET_MERGE</tt></a> </td></tr>
-<tr class="b">
+<tr class="a">
<td> <a href="#MULTISET_REMOVE"> <tt>MULTISET_REMOVE</tt></a> </td></tr>
-<tr class="a">
+<tr class="b">
<td> <a href="#MULTISET_TO_SET"> <tt>MULTISET_TO_SET</tt></a> </td></tr>
-<tr class="b">
-<td> <a href="#OBJECT_GET"> <tt>OBJECT_GET</tt></a> </td></tr>
<tr class="a">
-<td> <a href="#PREPEND_IF_MISSING"> <tt>PREPEND_IF_MISSING</tt></a> </td></tr>
+<td> <a href="#OBJECT_GET"> <tt>OBJECT_GET</tt></a> </td></tr>
<tr class="b">
+<td> <a href="#PREPEND_IF_MISSING"> <tt>PREPEND_IF_MISSING</tt></a> </td></tr>
+<tr class="a">
<td> <a href="#PROFILE_GET"> <tt>PROFILE_GET</tt></a> </td></tr>
+<tr class="b">
+<td> <a href="#PROFILE_VERBOSE"> <tt>PROFILE_VERBOSE</tt></a> </td></tr>
<tr class="a">
<td> <a href="#PROFILE_FIXED"> <tt>PROFILE_FIXED</tt></a> </td></tr>
<tr class="b">
@@ -511,9 +517,9 @@
<tr class="b">
<td> <a href="#REGEXP_GROUP_VAL"> <tt>REGEXP_GROUP_VAL</tt></a> </td></tr>
<tr class="a">
-<td> <a href="#REGEXP_REPLACE"> <tt>REGEXP_REPLACE</tt></a></td></tr>
+<td> <a href="#REGEXP_REPLACE"> <tt>REGEXP_REPLACE</tt></a> </td></tr>
<tr class="b">
-<td> <a href="#REST_GET"> <tt>REST_GET</tt></a></td></tr>
+<td> <a href="#REST_GET"> <tt>REST_GET</tt></a> </td></tr>
<tr class="a">
<td> <a href="#ROUND"> <tt>ROUND</tt></a> </td></tr>
<tr class="b">
@@ -660,6 +666,20 @@
<li>Returns: A new String if prefix was prepended, the same string otherwise.</li>
</ul></div>
<div class="section">
+<h3><a name="ASN_GET"></a><tt>ASN_GET</tt></h3>
+<ul>
+
+<li>Description: Look up an IPV4 address and returns Autonomous System Number information about it</li>
+<li>Input:
+<ul>
+
+<li>ip - The IPV4 address to lookup</li>
+<li>fields - Optional list of ASN fields to grab. Options are network, autonomous_system_number, autonomous_system_organization.</li>
+</ul>
+</li>
+<li>Returns: If a Single field is requested a string of the field, If multiple fields a map of string of the fields, and null otherwise</li>
+</ul></div>
+<div class="section">
<h3><a name="BLOOM_ADD"></a><tt>BLOOM_ADD</tt></h3>
<ul>
@@ -1600,6 +1620,34 @@
<li>Returns: The object associated with the key in the map. If no value is associated with the key and default is specified, then default is returned. If no value is associated with the key or default, then null is returned.</li>
</ul></div>
<div class="section">
+<h3><a name="MAP_MERGE"></a><tt>MAP_MERGE</tt></h3>
+<ul>
+
+<li>Description: Merges a list of maps</li>
+<li>Input:
+<ul>
+
+<li>maps - A collection of maps to merge. Last entry wins for overlapping keys.</li>
+</ul>
+</li>
+<li>Returns: A Map. null if the list of maps is empty.</li>
+</ul></div>
+<div class="section">
+<h3><a name="MAP_PUT"></a><tt>MAP_PUT</tt></h3>
+<ul>
+
+<li>Description: Adds a key/value pair to a map</li>
+<li>Input:
+<ul>
+
+<li>key - The key</li>
+<li>value - The value</li>
+<li>map - The map to perform the put on</li>
+</ul>
+</li>
+<li>Returns: The original map modified with the key/value. If the map argument is null, a new map will be created and returned that contains the provided key and value - note: if the ‘map’ argument is null, only the returned map will be non-null and contain the key/value.</li>
+</ul></div>
+<div class="section">
<h3><a name="MAX"></a><tt>MAX</tt></h3>
<ul>
@@ -1743,7 +1791,7 @@
<li>profile - The name of the profile.</li>
<li>entity - The name of the entity.</li>
-<li>periods - The list of profile periods to grab. These are ProfilePeriod objects.</li>
+<li>periods - The list of profile periods to fetch. Use PROFILE_WINDOW or PROFILE_FIXED.</li>
<li>groups_list - Optional, must correspond to the ‘groupBy’ list used in profile creation - List (in square brackets) of groupBy values used to filter the profile. Default is the empty list, meaning groupBy was not used when creating the profile.</li>
<li>config_overrides - Optional - Map (in curly braces) of name:value pairs, each overriding the global config parameter of the same name. Default is the empty Map, meaning no overrides.</li>
</ul>
@@ -1751,6 +1799,22 @@
<li>Returns: The selected profile measurements.</li>
</ul></div>
<div class="section">
+<h3><a name="PROFILE_VERBOSE"></a><tt>PROFILE_VERBOSE</tt></h3>
+<ul>
+
+<li>Description: Retrieves a series of measurements from a stored profile. Returns a map containing the profile name, entity, period id, period start, period end for each profile measurement. Provides a more verbose view of each measurement than PROFILE_GET.</li>
+<li>Input:
+<ul>
+
+<li>profile - The name of the profile.</li>
+<li>entity - The name of the entity.</li>
+<li>periods - The list of profile periods to fetch. Use PROFILE_WINDOW or PROFILE_FIXED.</li>
+<li>groups - Optional - The groups to retrieve. Must correspond to the ‘groupBy’ used during profile creation. Defaults to an empty list, meaning no groups.</li>
+</ul>
+</li>
+<li>Returns: The selected profile measurements.</li>
+</ul></div>
+<div class="section">
<h3><a name="PROFILE_FIXED"></a><tt>PROFILE_FIXED</tt></h3>
<ul>
@@ -2728,7 +2792,7 @@
<div>
<div>
-<pre class="source">metron-stellar/stellar-common/target/stellar-common-0.7.0-stand-alone.tar.gz
+<pre class="source">metron-stellar/stellar-common/target/stellar-common-0.7.1-stand-alone.tar.gz
</pre></div></div>
<p>When unpacked, the following structure will be created:</p>
@@ -2739,7 +2803,7 @@
├── bin
│   └── stellar
└── lib
- └── stellar-common-0.7.0-uber.jar
+ └── stellar-common-0.7.1-uber.jar
</pre></div></div>
<p>To run the Stellar Shell run the following from the directory you unpacked to:</p>
diff --git a/current-book/metron-stellar/stellar-zeppelin/index.html b/current-book/metron-stellar/stellar-zeppelin/index.html
index 2f3a3f6..30d5e76 100644
--- a/current-book/metron-stellar/stellar-zeppelin/index.html
+++ b/current-book/metron-stellar/stellar-zeppelin/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-zeppelin/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-zeppelin/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Stellar Interpreter for Apache Zeppelin</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Stellar Interpreter for Apache Zeppelin</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
@@ -132,12 +130,18 @@
<li>
<p>Use Zeppelin’s installation utility to install the Stellar Interpreter.</p>
-<p>If Zeppelin was already installed, make sure that it is stopped before running this command. Update the version, ‘0.7.0’ in the example below, to whatever is appropriate for your environment.</p>
+<p>If Zeppelin was already installed, make sure that it is stopped before running this command. Update the version, ‘0.7.1’ in the example below, to whatever is appropriate for your environment.</p>
<div>
<div>
-<pre class="source">bin/install-interpreter.sh --name stellar --artifact org.apache.metron:stellar-zeppelin:0.7.0
+<pre class="source">bin/install-interpreter.sh --name stellar --artifact org.apache.metron:stellar-zeppelin:0.7.1
</pre></div></div>
+
+<p><b>Note:</b> The above command will download maven artifact groupId1:artifact1:version1 (org.apache.metron:stellar-zeppelin:0.7.1) and all of its transitive dependencies into the $ZEPPELIN_HOME/interpreter/stellar directory. <tt>stellar-common</tt>, which contains many of the <a href="../stellar-common/index.html#Stellar_Core_Functions">Stellar Core Functions</a>, will be included transitively because <tt>stellar-zeppelin</tt> declares it as a direct dependency in its Maven pom.xml.</p>
+<ul>
+
+<li><a class="externalLink" href="https://zeppelin.apache.org/docs/0.7.3/manual/interpreterinstallation.html#3rd-party-interpreters">3rd Party Zeppelin Interpreter Installation Documentation</a></li>
+</ul>
</li>
<li>
@@ -208,7 +212,7 @@
<div>
<div>
-<pre class="source">org.apache.metron:metron-statistics:0.7.0
+<pre class="source">org.apache.metron:metron-statistics:0.7.1
</pre></div></div>
</li>
<li>
diff --git a/current-book/use-cases/forensic_clustering/index.html b/current-book/use-cases/forensic_clustering/index.html
index 4d0774d..cd3d3b7 100644
--- a/current-book/use-cases/forensic_clustering/index.html
+++ b/current-book/use-cases/forensic_clustering/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/forensic_clustering/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/forensic_clustering/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Problem Statement</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Problem Statement</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/use-cases/geographic_login_outliers/index.html b/current-book/use-cases/geographic_login_outliers/index.html
index 97a713d..4db5c80 100644
--- a/current-book/use-cases/geographic_login_outliers/index.html
+++ b/current-book/use-cases/geographic_login_outliers/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/geographic_login_outliers/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/geographic_login_outliers/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Problem Statement</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Problem Statement</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/use-cases/index.html b/current-book/use-cases/index.html
index 24b3d92..16b9a35 100644
--- a/current-book/use-cases/index.html
+++ b/current-book/use-cases/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Worked Examples</title>
<link rel="stylesheet" href="../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Worked Examples</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/use-cases/parser_chaining/index.html b/current-book/use-cases/parser_chaining/index.html
index f9667c5..c33d96f 100644
--- a/current-book/use-cases/parser_chaining/index.html
+++ b/current-book/use-cases/parser_chaining/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/parser_chaining/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/parser_chaining/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Problem Statement</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Problem Statement</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/current-book/use-cases/typosquat_detection/index.html b/current-book/use-cases/typosquat_detection/index.html
index 4ec0f7e..ec9f9cb 100644
--- a/current-book/use-cases/typosquat_detection/index.html
+++ b/current-book/use-cases/typosquat_detection/index.html
@@ -1,13 +1,13 @@
<!DOCTYPE html>
<!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/typosquat_detection/index.md at 2018-12-14
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/typosquat_detection/index.md at 2019-05-14
| Rendered using Apache Maven Fluido Skin 1.7
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20181214" />
+ <meta name="Date-Revision-yyyymmdd" content="20190514" />
<meta http-equiv="Content-Language" content="en" />
<title>Metron – Problem Statement</title>
<link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
<li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
<li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
<li class="active ">Problem Statement</li>
- <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-12-14</li>
- <li id="projectVersion" class="pull-right">Version: 0.7.0</li>
+ <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li>
+ <li id="projectVersion" class="pull-right">Version: 0.7.1</li>
</ul>
</div>
<div class="row-fluid">
@@ -49,9 +49,7 @@
<li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
<li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
<li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
- <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
- <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
- <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+ <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li>
<li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
<li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
<li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
diff --git a/documentation/index.html b/documentation/index.html
index dc2a42d..0d9e4a4 100644
--- a/documentation/index.html
+++ b/documentation/index.html
@@ -211,21 +211,21 @@
<section class="events-section info-block darken" id="releases">
<div class="wrapper downloads">
<div class="text-center">
- <h4>Current Release: 0.7.0</h4>
+ <h4>Current Release: 0.7.1</h4>
</div>
<div class="content-960 hover-btn text-center">
<ul>
<li>
- <a href="http://www.apache.org/dyn/closer.cgi/metron/0.7.0/apache-metron-0.7.0.tar.gz" target="new">
- apache-metron-0.7.0.tar.gz
+ <a href="http://www.apache.org/dyn/closer.cgi/metron/0.7.1/apache-metron_0.7.1.tar.gz" target="new">
+ apache-metron-0.7.1.tar.gz
</a>
- [ <a href="https://www.apache.org/dist/metron/0.7.0/apache-metron-0.7.0.tar.gz.asc" target="new">
+ [ <a href="https://www.apache.org/dist/metron/0.7.1/apache-metron_0.7.1.tar.gz.asc" target="new">
PGP
</a>]
- [ <a href="https://www.apache.org/dist/metron/0.7.0/apache-metron-0.7.0.tar.gz.sha256" target="new">
+ [ <a href="https://www.apache.org/dist/metron/0.7.1/apache-metron_0.7.1.tar.gz.sha256" target="new">
SHA256
</a>]
- [ <a href="https://www.apache.org/dist/metron/0.7.0/apache-metron-0.7.0.tar.gz.sha512" target="new">
+ [ <a href="https://www.apache.org/dist/metron/0.7.1/apache-metron_0.7.1.tar.gz.sha512" target="new">
SHA512
</a>]
[ <a href="/current-book/index.html" target="new"> Book </a>]
diff --git a/feed.xml b/feed.xml
index fa0a69a..92ae6ea 100644
--- a/feed.xml
+++ b/feed.xml
@@ -5,9 +5,9 @@
<description></description>
<link>http://localhost:4000/</link>
<atom:link href="http://localhost:4000/feed.xml" rel="self" type="application/rss+xml"/>
- <pubDate>Mon, 17 Dec 2018 09:15:48 -0500</pubDate>
- <lastBuildDate>Mon, 17 Dec 2018 09:15:48 -0500</lastBuildDate>
- <generator>Jekyll v3.8.3</generator>
+ <pubDate>Wed, 15 May 2019 17:22:18 -0400</pubDate>
+ <lastBuildDate>Wed, 15 May 2019 17:22:18 -0400</lastBuildDate>
+ <generator>Jekyll v3.8.5</generator>
<item>
<title>Debo Dutta joins PPMC</title>
