| <!DOCTYPE html> |
| <!-- |
| | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-management/index.md at 2019-05-14 |
| | Rendered using Apache Maven Fluido Skin 1.7 |
| --> |
| <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> |
| <head> |
| <meta charset="UTF-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
| <meta name="Date-Revision-yyyymmdd" content="20190514" /> |
| <meta http-equiv="Content-Language" content="en" /> |
| <title>Metron – Stellar REPL Management Utilities</title> |
| <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" /> |
| <link rel="stylesheet" href="../../css/site.css" /> |
| <link rel="stylesheet" href="../../css/print.css" media="print" /> |
| <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script> |
| <script type="text/javascript"> |
| $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } ); |
| </script> |
| </head> |
| <body class="topBarDisabled"> |
| <div class="container-fluid"> |
| <div id="banner"> |
| <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../images/metron-logo.png" alt="Apache Metron" width="148px" height="48px"/></a></div> |
| <div class="pull-right"></div> |
| <div class="clear"><hr/></div> |
| </div> |
| |
| <div id="breadcrumbs"> |
| <ul class="breadcrumb"> |
| <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> |
| <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li> |
| <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li> |
| <li class="active ">Stellar REPL Management Utilities</li> |
| <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2019-05-14</li> |
| <li id="projectVersion" class="pull-right">Version: 0.7.1</li> |
| </ul> |
| </div> |
| <div class="row-fluid"> |
| <div id="leftColumn" class="span2"> |
| <div class="well sidebar-nav"> |
| <ul class="nav nav-list"> |
| <li class="nav-header">User Documentation</li> |
| <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a> |
| <ul class="nav nav-list"> |
| <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li> |
| <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li> |
| <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li> |
| <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li> |
| <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li> |
| <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li> |
| <li><a href="../../metron-interface/index.html" title="Interface"><span class="icon-chevron-right"></span>Interface</a></li> |
| <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a> |
| <ul class="nav nav-list"> |
| <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li> |
| <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li> |
| <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li> |
| <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li> |
| <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li> |
| <li><a href="../../metron-platform/metron-hbase-server/index.html" title="Hbase-server"><span class="none"></span>Hbase-server</a></li> |
| <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li> |
| <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li> |
| <li class="active"><a href="#"><span class="none"></span>Management</a></li> |
| <li><a href="../../metron-platform/metron-parsing/index.html" title="Parsing"><span class="icon-chevron-right"></span>Parsing</a></li> |
| <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li> |
| <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li> |
| <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li> |
| </ul> |
| </li> |
| <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li> |
| <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li> |
| <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li> |
| <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li> |
| <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li> |
| </ul> |
| </li> |
| </ul> |
| <hr /> |
| <div id="poweredBy"> |
| <div class="clear"></div> |
| <div class="clear"></div> |
| <div class="clear"></div> |
| <div class="clear"></div> |
| <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a> |
| </div> |
| </div> |
| </div> |
| <div id="bodyColumn" class="span10" > |
| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one |
| or more contributor license agreements. See the NOTICE file |
| distributed with this work for additional information |
| regarding copyright ownership. The ASF licenses this file |
| to you under the Apache License, Version 2.0 (the |
| "License"); you may not use this file except in compliance |
| with the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| --> |
| <h1>Stellar REPL Management Utilities</h1> |
| <p><a name="Stellar_REPL_Management_Utilities"></a></p> |
| <p>In order to augment the functionality of the Stellar REPL, a few management functions surrounding the management of the configurations and the management of Stellar transformations in the following areas have been added:</p> |
| <ul> |
| |
| <li>Stellar field transformations in the Parsers</li> |
| <li>Stellar enrichments in the Enrichment topology</li> |
| <li>Stellar threat triage rules</li> |
| </ul> |
| <p>Additionally, some shell functions have been added to</p> |
| <ul> |
| |
| <li>provide the ability to refer to the Stellar expression used to create a variable</li> |
| <li>print structured data in a way that is easier to view (i.e. tabular)</li> |
| </ul> |
| <p>This functionality is exposed as a pack of Stellar functions in this project.</p> |
| <ul> |
| |
| <li><a href="#Functions">Functions</a> |
| <ul> |
| |
| <li><a href="#Grok_Functions">Grok Functions</a></li> |
| <li><a href="#File_Functions">File Functions</a></li> |
| <li><a href="#Configuration_Functions">Configuration Functions</a></li> |
| <li><a href="#Parser_Functions">Parser Functions</a></li> |
| <li><a href="#Indexing_Functions">Indexing Functions</a></li> |
| <li><a href="#Enrichment_Functions">Enrichment Functions</a></li> |
| <li><a href="#Threat_Triage_Functions">Threat Triage Functions</a></li> |
| </ul> |
| </li> |
| <li><a href="#Examples">Examples</a> |
| <ul> |
| |
| <li><a href="#Iterate_to_Find_a_Valid_Grok_Pattern">Iterate to Find a Valid Grok Pattern</a></li> |
| <li><a href="#Manage_Stellar_Field_Transformations">Manage Stellar Field Transformations</a></li> |
| <li><a href="#Manage_Stellar_Enrichments">Manage Stellar Enrichments</a></li> |
| <li><a href="#Manage_Threat_Triage_Rules">Manage Threat Triage Rules</a></li> |
| </ul> |
| </li> |
| </ul> |
| <div class="section"> |
| <h2><a name="Functions"></a>Functions</h2> |
| <p>The functions are split roughly into a few sections:</p> |
| <ul> |
| |
| <li>Shell functions - Functions surrounding interacting with the shell in either a nicer way or a more functional way.</li> |
| <li>Grok Functions - Functions that allow you to evaluate grok expressions.</li> |
| <li>File functions - Functions around interacting with local or HDFS files</li> |
| <li>Configuration functions - Functions surrounding pulling and pushing configs from zookeeper</li> |
| <li>Parser functions - Functions surrounding adding, viewing, and removing Parser functions.</li> |
| <li>Indexing functions - Functions surrounding setting indexing parameters (batch size and timeout), setting the index, enabling/disabling index writer for sensors.</li> |
| <li>Enrichment functions - Functions surrounding adding, viewing and removing Stellar enrichments as well as managing batch size, batch timeout, and index names for the enrichment topology configuration</li> |
| <li>Threat Triage functions - Functions surrounding adding, viewing and removing threat triage functions.</li> |
| </ul> |
| <table border="0" class="table table-striped"> |
| <thead> |
| |
| <tr class="a"> |
| <th align="center"> Type </th> |
| <th align="center"> Function Name </th></tr> |
| </thead><tbody> |
| |
| <tr class="b"> |
| <td align="center"> <b>Grok Functions</b> </td> |
| <td align="center"> <a href="#GROK_EVAL"><tt>GROK_EVAL</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#GROK_PREDICT"><tt>GROK_PREDICT</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> <b>File Functions (Local)</b> </td> |
| <td align="center"> <a href="#LOCAL_LS"><tt>LOCAL_LS</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#LOCAL_RM"><tt>LOCAL_RM</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#LOCAL_READ"><tt>LOCAL_READ</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> <b>File Functions (HDFS)</b> </td> |
| <td align="center"> <a href="#HDFS_LS"><tt>HDFS_LS</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#HDFS_RM"><tt>HDFS_RM</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#HDFS_READ"><tt>HDFS_READ</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> <b>Configuration Functions</b> </td> |
| <td align="center"> <a href="#CONFIG_GET"><tt>CONFIG_GET</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#CONFIG_PUT"><tt>CONFIG_PUT</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> <b>Parser Functions</b> </td> |
| <td align="center"> <a href="#PARSER_CONFIG"><tt>PARSER_CONFIG</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#PARSER_INIT"><tt>PARSER_INIT</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#PARSER_PARSE"><tt>PARSER_PARSE</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#PARSER_STELLAR_TRANSFORM_ADD"><tt>PARSER_STELLAR_TRANSFORM_ADD</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#PARSER_STELLAR_TRANSFORM_PRINT"><tt>PARSER_STELLAR_TRANSFORM_PRINT</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#PARSER_STELLAR_TRANSFORM_REMOVE"><tt>PARSER_STELLAR_TRANSFORM_REMOVE</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> <b>Threat Triage Functions</b> </td> |
| <td align="center"> <a href="#THREAT_TRIAGE_INIT"><tt>THREAT_TRIAGE_INIT</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#THREAT_TRIAGE_CONFIG"><tt>THREAT_TRIAGE_CONFIG</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#THREAT_TRIAGE_SCORE"><tt>THREAT_TRIAGE_SCORE</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#THREAT_TRIAGE_ADD"><tt>THREAT_TRIAGE_ADD</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#THREAT_TRIAGE_REMOVE"><tt>THREAT_TRIAGE_REMOVE</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#THREAT_TRIAGE_PRINT"><tt>THREAT_TRIAGE_PRINT</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#THREAT_TRIAGE_SET_AGGREGATOR"><tt>THREAT_TRIAGE_SET_AGGREGATOR</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> <b>Indexing Functions</b> </td> |
| <td align="center"> <a href="#INDEXING_SET_BATCH"><tt>INDEXING_SET_BATCH</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#INDEXING_SET_ENABLED"><tt>INDEXING_SET_ENABLED</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#INDEXING_SET_INDEX"><tt>INDEXING_SET_INDEX</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> <b>Enrichment Functions</b> </td> |
| <td align="center"> <a href="#ENRICHMENT_STELLAR_TRANSFORM_ADD"><tt>ENRICHMENT_STELLAR_TRANSFORM_ADD</tt></a> </td></tr> |
| <tr class="a"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#ENRICHMENT_STELLAR_TRANSFORM_PRINT"><tt>ENRICHMENT_STELLAR_TRANSFORM_PRINT</tt></a> </td></tr> |
| <tr class="b"> |
| <td align="center"> </td> |
| <td align="center"> <a href="#ENRICHMENT_STELLAR_TRANSFORM_REMOVE"><tt>ENRICHMENT_STELLAR_TRANSFORM_REMOVE</tt></a> </td></tr> |
| </tbody> |
| </table> |
| <div class="section"> |
| <h3><a name="Grok_Functions"></a>Grok Functions</h3> |
| <div class="section"> |
| <h4><a name="GROK_EVAL"></a><tt>GROK_EVAL</tt></h4> |
| <ul> |
| |
| <li>Description: Evaluate a grok expression for a statement.</li> |
| <li>Input: |
| <ul> |
| |
| <li>grokExpression - The grok expression to evaluate</li> |
| <li>data - Either a data message or a list of data messages to evaluate using the grokExpression</li> |
| </ul> |
| </li> |
| <li>Returns: The Map associated with the grok expression being evaluated on the list of messages.</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="GROK_PREDICT"></a><tt>GROK_PREDICT</tt></h4> |
| <ul> |
| |
| <li>Description: Discover a grok statement for an input doc</li> |
| <li>Input: |
| <ul> |
| |
| <li>data - The data to discover a grok expression from</li> |
| </ul> |
| </li> |
| <li>Returns: A grok expression that should match the data.</li> |
| </ul></div></div> |
| <div class="section"> |
| <h3><a name="File_Functions"></a>File Functions</h3> |
| <div class="section"> |
| <h4><a name="Local_Files"></a>Local Files</h4> |
| <div class="section"> |
| <h5><a name="LOCAL_LS"></a><tt>LOCAL_LS</tt></h5> |
| <ul> |
| |
| <li>Description: Lists the contents of a directory.</li> |
| <li>Input: |
| <ul> |
| |
| <li>path - The path of the file</li> |
| </ul> |
| </li> |
| <li>Returns: The contents of the directory in tabular form sorted by last modification date.</li> |
| </ul></div> |
| <div class="section"> |
| <h5><a name="LOCAL_RM"></a><tt>LOCAL_RM</tt></h5> |
| <ul> |
| |
| <li>Description: Removes the path</li> |
| <li>Input: |
| <ul> |
| |
| <li>path - The path of the file or directory.</li> |
| <li>recursive - Recursively remove or not (optional and defaulted to false)</li> |
| </ul> |
| </li> |
| <li>Returns: boolean - true if successful, false otherwise</li> |
| </ul></div> |
| <div class="section"> |
| <h5><a name="LOCAL_READ"></a><tt>LOCAL_READ</tt></h5> |
| <ul> |
| |
| <li>Description: Retrieves the contents as a string of a file.</li> |
| <li>Input: |
| <ul> |
| |
| <li>path - The path of the file</li> |
| </ul> |
| </li> |
| <li>Returns: The contents of the file and null otherwise.</li> |
| </ul></div> |
| <div class="section"> |
| <h5><a name="LOCAL_READ_LINES"></a><tt>LOCAL_READ_LINES</tt></h5> |
| <ul> |
| |
| <li>Description: Retrieves the contents of a file as a list of strings.</li> |
| <li>Input: |
| <ul> |
| |
| <li>path - The path of the file</li> |
| </ul> |
| </li> |
| <li>Returns: A list of lines</li> |
| </ul></div> |
| <div class="section"> |
| <h5><a name="LOCAL_WRITE"></a><tt>LOCAL_WRITE</tt></h5> |
| <ul> |
| |
| <li>Description: Writes the contents of a string to a local file</li> |
| <li>Input: |
| <ul> |
| |
| <li>content - The content to write out</li> |
| <li>path - The path of the file</li> |
| </ul> |
| </li> |
| <li>Returns: true if the file was written and false otherwise.</li> |
| </ul></div></div> |
| <div class="section"> |
| <h4><a name="HDFS_Files"></a>HDFS Files</h4> |
| <div class="section"> |
| <h5><a name="HDFS_LS"></a><tt>HDFS_LS</tt></h5> |
| <ul> |
| |
| <li>Description: Lists the contents of a directory in HDFS.</li> |
| <li>Input: |
| <ul> |
| |
| <li>path - The path of the file</li> |
| </ul> |
| </li> |
| <li>Returns: The contents of the directory in tabular form sorted by last modification date.</li> |
| </ul></div> |
| <div class="section"> |
| <h5><a name="HDFS_RM"></a><tt>HDFS_RM</tt></h5> |
| <ul> |
| |
| <li>Description: Removes the path in HDFS.</li> |
| <li>Input: |
| <ul> |
| |
| <li>path - The path of the file or directory.</li> |
| <li>recursive - Recursively remove or not (optional and defaulted to false)</li> |
| </ul> |
| </li> |
| <li>Returns: boolean - true if successful, false otherwise</li> |
| </ul></div> |
| <div class="section"> |
| <h5><a name="HDFS_READ"></a><tt>HDFS_READ</tt></h5> |
| <ul> |
| |
| <li>Description: Retrieves the contents as a string of a file in HDFS.</li> |
| <li>Input: |
| <ul> |
| |
| <li>path - The path of the file</li> |
| </ul> |
| </li> |
| <li>Returns: The contents of the file and null otherwise.</li> |
| </ul></div> |
| <div class="section"> |
| <h5><a name="HDFS_READ_LINES"></a><tt>HDFS_READ_LINES</tt></h5> |
| <ul> |
| |
| <li>Description: Retrieves the contents of a HDFS file as a list of strings.</li> |
| <li>Input: |
| <ul> |
| |
| <li>path - The path of the file</li> |
| </ul> |
| </li> |
| <li>Returns: A list of lines</li> |
| </ul></div> |
| <div class="section"> |
| <h5><a name="HDFS_WRITE"></a><tt>HDFS_WRITE</tt></h5> |
| <ul> |
| |
| <li>Description: Writes the contents of a string to a HDFS file</li> |
| <li>Input: |
| <ul> |
| |
| <li>content - The content to write out</li> |
| <li>path - The path of the file</li> |
| </ul> |
| </li> |
| <li>Returns: true if the file was written and false otherwise.</li> |
| </ul></div></div></div> |
| <div class="section"> |
| <h3><a name="Configuration_Functions"></a>Configuration Functions</h3> |
| <div class="section"> |
| <h4><a name="CONFIG_GET"></a><tt>CONFIG_GET</tt></h4> |
| <ul> |
| |
| <li>Description: Retrieve a Metron configuration from zookeeper.</li> |
| <li>Input: |
| <ul> |
| |
| <li>type - One of ENRICHMENT, INDEXING, PARSER, GLOBAL, PROFILER</li> |
| <li>sensor - Sensor to retrieve (required for enrichment and parser, not used for profiler and global)</li> |
| <li>emptyIfNotPresent - If true, then return an empty, minimally viable config</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the config in zookeeper</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="CONFIG_PUT"></a><tt>CONFIG_PUT</tt></h4> |
| <ul> |
| |
| <li>Description: Updates a Metron config to Zookeeper.</li> |
| <li>Input: |
| <ul> |
| |
| <li>type - One of ENRICHMENT, INDEXING, PARSER, GLOBAL, PROFILER</li> |
| <li>config - The config (a string in JSON form) to update</li> |
| <li>sensor - Sensor to retrieve (required for enrichment and parser, not used for profiler and global)</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the config in zookeeper</li> |
| </ul></div></div> |
| <div class="section"> |
| <h3><a name="Parser_Functions"></a>Parser Functions</h3> |
| <div class="section"> |
| <h4><a name="PARSER_CONFIG"></a><tt>PARSER_CONFIG</tt></h4> |
| <ul> |
| |
| <li>Description: Returns the configuration of the parser.</li> |
| <li>Input: |
| <ul> |
| |
| <li>parser - The parser created with PARSER_INIT.</li> |
| </ul> |
| </li> |
| <li>Returns: The parser configuration.</li> |
| </ul> |
| <div class="section"> |
| <h5><a name="Example"></a>Example</h5> |
| <ol style="list-style-type: decimal"> |
| |
| <li> |
| |
| <p>Initialize the parser. The parser configuration for the sensor ‘bro’ will be loaded automatically from Zookeeper.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> parser := PARSER_INIT("bro") |
| Parser{0 successful, 0 error(s)} |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Review the parser configuration for the ‘bro’ sensor.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> PARSER_CONFIG(parser) |
| { |
| "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser", |
| "filterClassName":"org.apache.metron.parsers.filters.StellarFilter", |
| "sensorTopic":"bro" |
| } |
| </pre></div></div> |
| </li> |
| </ol></div></div> |
| <div class="section"> |
| <h4><a name="PARSER_INIT"></a><tt>PARSER_INIT</tt></h4> |
| <ul> |
| |
| <li>Initialize a parser to parse the raw telemetry produced by a sensor.</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorType - The type of sensor to parse.</li> |
| <li>config - [Optional] The parser configuration. If not provided, the configuration will be retrieved from Zookeeper.</li> |
| </ul> |
| </li> |
| <li>Returns: A parser that can be used to parse sensor telemetry with <tt>PARSER_PARSE</tt>.</li> |
| </ul> |
| <div class="section"> |
| <h5><a name="Example"></a>Example</h5> |
| <ol style="list-style-type: decimal"> |
| |
| <li> |
| |
| <p>Initialize a parser. The parser configuration for the sensor ‘bro’ will be loaded automatically from Zookeeper.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> parser := PARSER_INIT("bro") |
| Parser{0 successful, 0 error(s)} |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>The parser will track the number of successes and failures and echo that information.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> parser |
| Parser{0 successful, 0 error(s)} |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Alternatively, explicitly define the parser configuration. This can be useful when creating a new parser or testing changes to an existing parser.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> config := SHELL_EDIT() |
| { |
| "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser", |
| "filterClassName":"org.apache.metron.parsers.filters.StellarFilter", |
| "sensorTopic":"bro" |
| } |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Initialize the parser with the parser configuration.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> parser := PARSER_INIT("bro", config) |
| Parser{0 successful, 0 error(s)} |
| </pre></div></div> |
| </li> |
| </ol></div></div> |
| <div class="section"> |
| <h4><a name="PARSER_PARSE"></a><tt>PARSER_PARSE</tt></h4> |
| <ul> |
| |
| <li>Parses the raw telemetry produced by a sensor.</li> |
| <li>Input: |
| <ul> |
| |
| <li>parser - The parser created with PARSER_INIT.</li> |
| <li>input - A telemetry message or list of messages to parse.</li> |
| </ul> |
| </li> |
| <li>Returns: A list of messages that result from parsing the input telemetry. If the input cannot be parsed, a message encapsulating the error is returned as part of that list.</li> |
| </ul> |
| <div class="section"> |
| <h5><a name="Example"></a>Example</h5> |
| <ol style="list-style-type: decimal"> |
| |
| <li> |
| |
| <p>Initialize the parser. The parser configuration for the sensor ‘bro’ will be loaded automatically from Zookeeper.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> parser := PARSER_INIT("bro") |
| Parser{0 successful, 0 error(s)} |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Grab the raw telemetry from the ‘bro’ input topic. You could also mock-up a string that you would like to parse using <tt>SHELL_EDIT</tt>.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> to_parse := KAFKA_GET('bro') |
| [{"http": {"ts":1542313125.807068,"uid":"CUrRne3iLIxXavQtci","id.orig_h"... |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Parse the telemetry.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> msgs := PARSER_PARSE(parser, to_parse) |
| [{"bro_timestamp":"1542313125.807068","method":"GET","ip_dst_port":8080,... |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>The parser will tally the success.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> parser |
| Parser{1 successful, 0 error(s)} |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Review the successfully parsed message.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> LENGTH(msgs) |
| 1 |
| </pre></div></div> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> msg := GET(msgs, 0) |
| |
| [Stellar]>>> MAP_GET("guid", msg) |
| 7f2e0c77-c58c-488e-b1ad-fbec10fb8182 |
| </pre></div></div> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> MAP_GET("timestamp", msg) |
| 1542313125807 |
| </pre></div></div> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> MAP_GET("source.type", msg) |
| bro |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Attempt to parse invalid input. This will return the error message that is pushed onto the error topic. The error message contains all of the details indicating why parsing failed.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> errors := PARSER_PARSE(parser, "{invalid>") |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Review the details of the error.</p> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> error := GET(errors, 0) |
| </pre></div></div> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> MAP_GET("raw_message", error) |
| {invalid> |
| </pre></div></div> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> MAP_GET("message", error) |
| Unable to parse Message: {invalid> |
| </pre></div></div> |
| |
| <div> |
| <div> |
| <pre class="source"> [Stellar]>>> MAP_GET("stack", error) |
| java.lang.IllegalStateException: Unable to parse Message: {invalid> |
| at org.apache.metron.parsers.bro.BasicBroParser.parse(BasicBroParser.java:145) |
| ... |
| </pre></div></div> |
| </li> |
| </ol></div></div> |
| <div class="section"> |
| <h4><a name="PARSER_STELLAR_TRANSFORM_ADD"></a><tt>PARSER_STELLAR_TRANSFORM_ADD</tt></h4> |
| <ul> |
| |
| <li>Description: Add stellar field transformation.</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| <li>stellarTransforms - A Map associating fields to stellar expressions</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the config in zookeeper</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="PARSER_STELLAR_TRANSFORM_PRINT"></a><tt>PARSER_STELLAR_TRANSFORM_PRINT</tt></h4> |
| <ul> |
| |
| <li>Description: Retrieve stellar field transformations.</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the transformations</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="PARSER_STELLAR_TRANSFORM_REMOVE"></a><tt>PARSER_STELLAR_TRANSFORM_REMOVE</tt></h4> |
| <ul> |
| |
| <li>Description: Remove stellar field transformation.</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| <li>stellarTransforms - A list of stellar transforms to remove</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the config in zookeeper</li> |
| </ul></div></div> |
| <div class="section"> |
| <h3><a name="Indexing_Functions"></a>Indexing Functions</h3> |
| <div class="section"> |
| <h4><a name="INDEXING_SET_BATCH"></a><tt>INDEXING_SET_BATCH</tt></h4> |
| <ul> |
| |
| <li>Description: Set batch size and timeout</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| <li>writer - The writer to update (e.g. elasticsearch, solr or hdfs)</li> |
| <li>size - batch size (integer), defaults to 1, meaning batching disabled</li> |
| <li>timeout - (optional) batch timeout in seconds (integer), defaults to 0, meaning system default</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the config in zookeeper</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="INDEXING_SET_ENABLED"></a><tt>INDEXING_SET_ENABLED</tt></h4> |
| <ul> |
| |
| <li>Description: Enable or disable an indexing writer for a sensor.</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| <li>writer - The writer to update (e.g. elasticsearch, solr or hdfs)</li> |
| <li>enabled? - boolean indicating whether the writer is enabled. If omitted, then it will set enabled.</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the config in zookeeper</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="INDEXING_SET_INDEX"></a><tt>INDEXING_SET_INDEX</tt></h4> |
| <ul> |
| |
| <li>Description: Set the index for the sensor</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| <li>writer - The writer to update (e.g. elasticsearch, solr or hdfs)</li> |
| <li>sensor - sensor name</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the config in zookeeper</li> |
| </ul></div></div> |
| <div class="section"> |
| <h3><a name="Enrichment_Functions"></a>Enrichment Functions</h3> |
| <div class="section"> |
| <h4><a name="ENRICHMENT_STELLAR_TRANSFORM_ADD"></a><tt>ENRICHMENT_STELLAR_TRANSFORM_ADD</tt></h4> |
| <ul> |
| |
| <li>Description: Add stellar field transformation.</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| <li>type - ENRICHMENT or THREAT_INTEL</li> |
| <li>stellarTransforms - A Map associating fields to stellar expressions</li> |
| <li>group - Group to add to (optional)</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the config in zookeeper</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="ENRICHMENT_STELLAR_TRANSFORM_PRINT"></a><tt>ENRICHMENT_STELLAR_TRANSFORM_PRINT</tt></h4> |
| <ul> |
| |
| <li>Description: Retrieve stellar enrichment transformations.</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| <li>type - ENRICHMENT or THREAT_INTEL</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the transformations</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="ENRICHMENT_STELLAR_TRANSFORM_REMOVE"></a><tt>ENRICHMENT_STELLAR_TRANSFORM_REMOVE</tt></h4> |
| <ul> |
| |
| <li>Description: Remove one or more stellar field transformations.</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| <li>type - ENRICHMENT or THREAT_INTEL</li> |
| <li>stellarTransforms - A list of removals</li> |
| <li>group - Group to remove from (optional)</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the config in zookeeper</li> |
| </ul></div></div> |
| <div class="section"> |
| <h3><a name="Threat_Triage_Functions"></a>Threat Triage Functions</h3> |
| <div class="section"> |
| <h4><a name="THREAT_TRIAGE_INIT"></a><tt>THREAT_TRIAGE_INIT</tt></h4> |
| <ul> |
| |
| <li>Description: Create a threat triage engine.</li> |
| <li>Input: |
| <ul> |
| |
| <li>config - the threat triage configuration (optional)</li> |
| </ul> |
| </li> |
| <li>Returns: A threat triage engine.</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="THREAT_TRIAGE_CONFIG"></a><tt>THREAT_TRIAGE_CONFIG</tt></h4> |
| <ul> |
| |
| <li>Description: Export the configuration used by a threat triage engine.</li> |
| <li>Input: |
| <ul> |
| |
| <li>engine - threat triage engine returned by THREAT_TRIAGE_INIT.</li> |
| </ul> |
| </li> |
| <li>Returns: The configuration used by the threat triage engine.</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="THREAT_TRIAGE_SCORE"></a><tt>THREAT_TRIAGE_SCORE</tt></h4> |
| <ul> |
| |
| <li>Description: Scores a message using a set of triage rules.</li> |
| <li>Inputs: |
| <ul> |
| |
| <li>message - a string containing the message to score.</li> |
| <li>engine - threat triage engine returned by THREAT_TRIAGE_INIT.</li> |
| </ul> |
| </li> |
| <li>Returns: A threat triage engine.</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="THREAT_TRIAGE_ADD"></a><tt>THREAT_TRIAGE_ADD</tt></h4> |
| <ul> |
| |
| <li>Description: Add a threat triage rule.</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| <li>stellarTransforms - A Map associating stellar rules to scores</li> |
| <li>triageRules - Map (or list of Maps) representing a triage rule. It must contain ‘rule’ and ‘score’ keys, the stellar expression for the rule and triage score respectively. It may contain ‘name’ and ‘comment’, the name of the rule and comment associated with the rule respectively."</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the threat triage rules</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="THREAT_TRIAGE_REMOVE"></a><tt>THREAT_TRIAGE_REMOVE</tt></h4> |
| <ul> |
| |
| <li>Description: Remove stellar threat triage rule(s).</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| <li>rules - A list of stellar rules or rule names to remove</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the enrichment config</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="THREAT_TRIAGE_PRINT"></a><tt>THREAT_TRIAGE_PRINT</tt></h4> |
| <ul> |
| |
| <li>Description: Retrieve stellar enrichment transformations.</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the threat triage rules</li> |
| </ul></div> |
| <div class="section"> |
| <h4><a name="THREAT_TRIAGE_SET_AGGREGATOR"></a><tt>THREAT_TRIAGE_SET_AGGREGATOR</tt></h4> |
| <ul> |
| |
| <li>Description: Set the threat triage aggregator.</li> |
| <li>Input: |
| <ul> |
| |
| <li>sensorConfig - Sensor config to add transformation to.</li> |
| <li>aggregator - Aggregator to use. One of MIN, MAX, MEAN, SUM, POSITIVE_MEAN</li> |
| <li>aggregatorConfig - Optional config for aggregator</li> |
| </ul> |
| </li> |
| <li>Returns: The String representation of the enrichment config</li> |
| </ul> |
| <div class="section"> |
| <h5><a name="Example"></a>Example</h5> |
| <ol style="list-style-type: decimal"> |
| |
| <li> |
| |
| <p>Create a threat triage engine.</p> |
| |
| <div> |
| <div> |
| <pre class="source">[Stellar]>>> t := THREAT_TRIAGE_INIT() |
| [Stellar]>>> t |
| ThreatTriage{0 rule(s)} |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Add a few triage rules.</p> |
| |
| <div> |
| <div> |
| <pre class="source">[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule1", "rule":"value>10", "score":10}) |
| </pre></div></div> |
| |
| <div> |
| <div> |
| <pre class="source">[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule2", "rule":"value>20", "score":20}) |
| </pre></div></div> |
| |
| <div> |
| <div> |
| <pre class="source">[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule3", "rule":"value>30", "score":30}) |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Review the rules that you have created.</p> |
| |
| <div> |
| <div> |
| <pre class="source">[Stellar]>>> THREAT_TRIAGE_PRINT(t) |
| ╔═══════╤═════════╤═════════════╤═══════╤════════╗ |
| ║ Name │ Comment │ Triage Rule │ Score │ Reason ║ |
| ╠═══════╪═════════╪═════════════╪═══════╪════════╣ |
| ║ rule1 │ │ value>10 │ 10 │ ║ |
| ╟───────┼─────────┼─────────────┼───────┼────────╢ |
| ║ rule2 │ │ value>20 │ 20 │ ║ |
| ╟───────┼─────────┼─────────────┼───────┼────────╢ |
| ║ rule3 │ │ value>30 │ 30 │ ║ |
| ╚═══════╧═════════╧═════════════╧═══════╧════════╝ |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Create a few test messages to simulate your telemetry.</p> |
| |
| <div> |
| <div> |
| <pre class="source">[Stellar]>>> msg1 := "{ \"value\":22 }" |
| [Stellar]>>> msg1 |
| { "value":22 } |
| </pre></div></div> |
| |
| <div> |
| <div> |
| <pre class="source">[Stellar]>>> msg2 := "{ \"value\":44 }" |
| [Stellar]>>> msg2 |
| { "value":44 } |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>Score a message based on the rules that have been defined. The result allows you to see the total score, the aggregator, along with details about each rule that fired.</p> |
| |
| <div> |
| <div> |
| <pre class="source">[Stellar]>>> THREAT_TRIAGE_SCORE( msg1, t) |
| {score=20.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value>10}, {score=20.0, name=rule2, rule=value>20}]} |
| </pre></div></div> |
| |
| <div> |
| <div> |
| <pre class="source">[Stellar]>>> THREAT_TRIAGE_SCORE( msg2, t) |
| {score=30.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value>10}, {score=20.0, name=rule2, rule=value>20}, {score=30.0, name=rule3, rule=value>30}]} |
| </pre></div></div> |
| </li> |
| <li> |
| |
| <p>From here you can iterate on your rule set until it does exactly what you need it to do. Once you have a working rule set, extract the configuration and push it into your live, Metron cluster.</p> |
| |
| <div> |
| <div> |
| <pre class="source">[Stellar]>>> conf := THREAT_TRIAGE_CONFIG( t) |
| [Stellar]>>> conf |
| { |
| "enrichment" : { |
| "fieldMap" : { }, |
| "fieldToTypeMap" : { }, |
| "config" : { } |
| }, |
| "threatIntel" : { |
| "fieldMap" : { }, |
| "fieldToTypeMap" : { }, |
| "config" : { }, |
| "triageConfig" : { |
| "riskLevelRules" : [ { |
| "name" : "rule1", |
| "rule" : "value>10", |
| "score" : 10.0 |
| }, { |
| "name" : "rule2", |
| "rule" : "value>20", |
| "score" : 20.0 |
| }, { |
| "name" : "rule3", |
| "rule" : "value>30", |
| "score" : 30.0 |
| }], |
| "aggregator" : "MAX", |
| "aggregationConfig" : { } |
| } |
| }, |
| "configuration" : { } |
| } |
| </pre></div></div> |
| |
| <div> |
| <div> |
| <pre class="source">[Stellar]>>> CONFIG_PUT("ENRICHMENT", conf, "bro") |
| </pre></div></div> |
| </li> |
| </ol></div></div></div></div> |
| <div class="section"> |
| <h2><a name="Deployment_Instructions"></a>Deployment Instructions</h2> |
| <ul> |
| |
| <li>Clusters installed via Ambari Management Pack (default) |
| <ul> |
| |
| <li>Automatically deployed</li> |
| </ul> |
| </li> |
| <li>Manual installation |
| <ul> |
| |
| <li>Deployment is as simple as dropping the jar created by this project into <tt>$METRON_HOME/lib</tt> and starting the Stellar shell via <tt>$METRON_HOME/bin/stellar</tt></li> |
| </ul> |
| </li> |
| </ul></div> |
| <div class="section"> |
| <h2><a name="Examples"></a>Examples</h2> |
| <p>Included for description and education purposes are a couple example Stellar REPL transcripts with helpful comments to illustrate some common operations.</p> |
| <div class="section"> |
| <h3><a name="Iterate_to_Find_a_Valid_Grok_pattern"></a>Iterate to Find a Valid Grok pattern</h3> |
| |
| <div> |
| <div> |
| <pre class="source">Stellar, Go! |
| Please note that functions are loading lazily in the background and will be unavailable until loaded fully. |
| [Stellar]>>> # We are going to debug a squid grok statement with a bug in it |
| [Stellar]>>> squid_grok_orig := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} |
| - %{WORD:UNWANTED}/%{IP:ip_dst_addr} %{WORD:UNWANTED}/%{WORD:UNWANTED}' |
| [Stellar]>>> # We have gone ot a couple of domains in squid: |
| [Stellar]>>> # 1475022887.362 256 127.0.0.1 TCP_MISS/301 803 GET http://www.youtube.com/ - DIRECT/216.58.216.238 text/html |
| [Stellar]>>> # 1475022915.731 1 127.0.0.1 NONE/400 3520 GET flimflammadeupdomain.com - NONE/- text/html |
| [Stellar]>>> # 1475022938.661 0 127.0.0.1 NONE/400 3500 GET www.google.com - NONE/- text/html |
| [Stellar]>>> # Note that flimflammadeupdomain.com and www.google.com did not resolve to IPs |
| [Stellar]>>> # We can load up these messages from disk into a list of messages |
| [Stellar]>>> messages := LOCAL_READ_LINES( '/var/log/squid/access.log') |
| 27687 [Thread-1] INFO o.r.Reflections - Reflections took 26542 ms to scan 22 urls, producing 17906 keys and 121560 values |
| 27837 [Thread-1] INFO o.a.m.c.d.FunctionResolverSingleton - Found 97 Stellar Functions... |
| Functions loaded, you may refer to functions now... |
| [Stellar]>>> # and evaluate the messages against our grok statement |
| [Stellar]>>> GROK_EVAL(squid_grok_orig, messages) |
| ╔══════════╤═════════╤═════════╤═════════╤════════════════╤═════════════╤═════════╤════════════════╤═════════════════════════╗ |
| ║ action │ bytes │ code │ elapsed │ ip_dst_addr │ ip_src_addr │ method │ timestamp │ url ║ |
| ╠══════════╪═════════╪═════════╪═════════╪════════════════╪═════════════╪═════════╪════════════════╪═════════════════════════╣ |
| ║ TCP_MISS │ 803 │ 301 │ 256 │ 216.58.216.238 │ 127.0.0.1 │ GET │ 1475022887.362 │ http://www.youtube.com/ ║ |
| ╟──────────┼─────────┼─────────┼─────────┼────────────────┼─────────────┼─────────┼────────────────┼─────────────────────────╢ |
| ║ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING ║ |
| ╟──────────┼─────────┼─────────┼─────────┼────────────────┼─────────────┼─────────┼────────────────┼─────────────────────────╢ |
| ║ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING ║ |
| ╚══════════╧═════════╧═════════╧═════════╧════════════════╧═════════════╧═════════╧════════════════╧═════════════════════════╝ |
| |
| [Stellar]>>> # Uh oh, looks like the messages without destination IPs do not parse |
| [Stellar]>>> # We can start peeling off groups from the end of the message until things parse |
| [Stellar]>>> squid_grok := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{ |
| WORD:UNWANTED}/%{IP:ip_dst_addr}' |
| [Stellar]>>> GROK_EVAL(squid_grok, messages) |
| ╔══════════╤═════════╤═════════╤═════════╤════════════════╤═════════════╤═════════╤════════════════╤═════════════════════════╗ |
| ║ action │ bytes │ code │ elapsed │ ip_dst_addr │ ip_src_addr │ method │ timestamp │ url ║ |
| ╠══════════╪═════════╪═════════╪═════════╪════════════════╪═════════════╪═════════╪════════════════╪═════════════════════════╣ |
| ║ TCP_MISS │ 803 │ 301 │ 256 │ 216.58.216.238 │ 127.0.0.1 │ GET │ 1475022887.362 │ http://www.youtube.com/ ║ |
| ╟──────────┼─────────┼─────────┼─────────┼────────────────┼─────────────┼─────────┼────────────────┼─────────────────────────╢ |
| ║ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING ║ |
| ╟──────────┼─────────┼─────────┼─────────┼────────────────┼─────────────┼─────────┼────────────────┼─────────────────────────╢ |
| ║ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING ║ |
| ╚══════════╧═════════╧═════════╧═════════╧════════════════╧═════════════╧═════════╧════════════════╧═════════════════════════╝ |
| |
| [Stellar]>>> # Still looks like it is having issues... |
| [Stellar]>>> squid_grok := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{ |
| WORD:UNWANTED}/%{IP:ip_dst_addr}' |
| [Stellar]>>> GROK_EVAL(squid_grok, messages) |
| ╔══════════╤═════════╤═════════╤═════════╤════════════════╤═════════════╤═════════╤════════════════╤═════════════════════════╗ |
| ║ action │ bytes │ code │ elapsed │ ip_dst_addr │ ip_src_addr │ method │ timestamp │ url ║ |
| ╠══════════╪═════════╪═════════╪═════════╪════════════════╪═════════════╪═════════╪════════════════╪═════════════════════════╣ |
| ║ TCP_MISS │ 803 │ 301 │ 256 │ 216.58.216.238 │ 127.0.0.1 │ GET │ 1475022887.362 │ http://www.youtube.com/ ║ |
| ╟──────────┼─────────┼─────────┼─────────┼────────────────┼─────────────┼─────────┼────────────────┼─────────────────────────╢ |
| ║ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING ║ |
| ╟──────────┼─────────┼─────────┼─────────┼────────────────┼─────────────┼─────────┼────────────────┼─────────────────────────╢ |
| ║ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING │ MISSING ║ |
| ╚══════════╧═════════╧═════════╧═════════╧════════════════╧═════════════╧═════════╧════════════════╧═════════════════════════╝ |
| |
| [Stellar]>>> # Still looks wrong. Hmm, I bet it is due to that dst_addr not being there; we can make it optional |
| [Stellar]>>> squid_grok := '%{NUMBER:timestamp} %{SPACE:UNWANTED} %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{ |
| WORD:UNWANTED}/(%{IP:ip_dst_addr})?' |
| [Stellar]>>> GROK_EVAL(squid_grok, messages) |
| ╔══════════╤═══════╤══════╤═════════╤════════════════╤═════════════╤════════╤════════════════╤══════════════════════════╗ |
| ║ action │ bytes │ code │ elapsed │ ip_dst_addr │ ip_src_addr │ method │ timestamp │ url ║ |
| ╠══════════╪═══════╪══════╪═════════╪════════════════╪═════════════╪════════╪════════════════╪══════════════════════════╣ |
| ║ TCP_MISS │ 803 │ 301 │ 256 │ 216.58.216.238 │ 127.0.0.1 │ GET │ 1475022887.362 │ http://www.youtube.com/ ║ |
| ╟──────────┼───────┼──────┼─────────┼────────────────┼─────────────┼────────┼────────────────┼──────────────────────────╢ |
| ║ NONE │ 3520 │ 400 │ 1 │ null │ 127.0.0.1 │ GET │ 1475022915.731 │ flimflammadeupdomain.com ║ |
| ╟──────────┼───────┼──────┼─────────┼────────────────┼─────────────┼────────┼────────────────┼──────────────────────────╢ |
| ║ NONE │ 3500 │ 400 │ 0 │ null │ 127.0.0.1 │ GET │ 1475022938.661 │ www.google.com ║ |
| ╚══════════╧═══════╧══════╧═════════╧════════════════╧═════════════╧════════╧════════════════╧══════════════════════════╝ |
| |
| [Stellar]>>> # Ahh, that is much better. |
| [Stellar]>>> |
| [Stellar]>>> |
| </pre></div></div> |
| </div> |
| <div class="section"> |
| <h3><a name="Manage_Stellar_Field_Transformations"></a>Manage Stellar Field Transformations</h3> |
| |
| <div> |
| <div> |
| <pre class="source">964 [main] INFO o.a.c.f.i.CuratorFrameworkImpl - Starting |
| 1025 [main-EventThread] INFO o.a.c.f.s.ConnectionStateManager - State change: CONNECTED |
| Stellar, Go! |
| Please note that functions are loading lazily in the background and will be unavailable until loaded fully. |
| {es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH} |
| [Stellar]>>> # First we get the squid parser config from zookeeper |
| [Stellar]>>> squid_parser_config := CONFIG_GET('PARSER', 'squid') |
| 29089 [Thread-1] INFO o.r.Reflections - Reflections took 26765 ms to scan 22 urls, producing 17898 keys and 121518 values |
| 29177 [Thread-1] INFO o.a.m.c.d.FunctionResolverSingleton - Found 83 Stellar Functions... |
| Functions loaded, you may refer to functions now... |
| [Stellar]>>> # See what kind of transformations it already has |
| [Stellar]>>> PARSER_STELLAR_TRANSFORM_PRINT(squid_parser_config) |
| ╔═══════════════════════════╤═════════════════════════════════════════╗ |
| ║ Field │ Transformation ║ |
| ╠═══════════════════════════╪═════════════════════════════════════════╣ |
| ║ full_hostname │ URL_TO_HOST(url) ║ |
| ╟───────────────────────────┼─────────────────────────────────────────╢ |
| ║ domain_without_subdomains │ DOMAIN_REMOVE_SUBDOMAINS(full_hostname) ║ |
| ╚═══════════════════════════╧═════════════════════════════════════════╝ |
| |
| [Stellar]>>> #Just to make sure it looks right, we can view the JSON |
| [Stellar]>>> squid_parser_config |
| { |
| "parserClassName": "org.apache.metron.parsers.GrokParser", |
| "sensorTopic": "squid", |
| "parserConfig": { |
| "grokPath": "/patterns/squid", |
| "patternLabel": "SQUID_DELIMITED", |
| "timestampField": "timestamp" |
| }, |
| "fieldTransformations" : [ |
| { |
| "transformation" : "STELLAR" |
| ,"output" : [ "full_hostname", "domain_without_subdomains" ] |
| ,"config" : { |
| "full_hostname" : "URL_TO_HOST(url)" |
| ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" |
| } |
| } |
| ] |
| } |
| |
| [Stellar]>>> # Add another transformation in there |
| [Stellar]>>> domain_without_subdomains := 'cnn.com' |
| [Stellar]>>> upper_domain := TO_UPPER(domain_without_subdomains) |
| [Stellar]>>> # Now we can look at our variables and see what expressions created them |
| [Stellar]>>> # NOTE: the 40 is the max char for a word |
| [Stellar]>>> SHELL_LIST_VARS( 40 ) |
| ╔═══════════════════════════╤═══════════════════════════════════════════╤═════════════════════════════════════╗ |
| ║ VARIABLE │ VALUE │ EXPRESSION ║ |
| ╠═══════════════════════════╪═══════════════════════════════════════════╪═════════════════════════════════════╣ |
| ║ squid_parser_config │ { │ CONFIG_GET('PARSER', 'squid') ║ |
| ║ │ "parserClassName": │ ║ |
| ║ │ "org.apache.metron.parsers.GrokParser", │ ║ |
| ║ │ │ ║ |
| ║ │ "sensorTopic": "squid", │ ║ |
| ║ │ │ ║ |
| ║ │ "parserConfig": { │ ║ |
| ║ │ "grokPath": │ ║ |
| ║ │ "/patterns/squid", │ ║ |
| ║ │ "patternLabel": │ ║ |
| ║ │ "SQUID_DELIMITED", │ ║ |
| ║ │ "timestampField": │ ║ |
| ║ │ "timestamp" │ ║ |
| ║ │ }, │ ║ |
| ║ │ │ ║ |
| ║ │ "fieldTransformations" : [ │ ║ |
| ║ │ { │ ║ |
| ║ │ │ ║ |
| ║ │ "transformation" : "STELLAR" │ ║ |
| ║ │ │ ║ |
| ║ │ ,"output" : [ "full_hostname", │ ║ |
| ║ │ "domain_without_subdomains" ] │ ║ |
| ║ │ │ ║ |
| ║ │ ,"config" : { │ ║ |
| ║ │ "full_hostname" : │ ║ |
| ║ │ "URL_TO_HOST(url)" │ ║ |
| ║ │ │ ║ |
| ║ │ ,"domain_without_subdomains" : │ ║ |
| ║ │ "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" │ ║ |
| ║ │ │ ║ |
| ║ │ } │ ║ |
| ║ │ } │ ║ |
| ║ │ ] │ ║ |
| ║ │ } │ ║ |
| ╟───────────────────────────┼───────────────────────────────────────────┼─────────────────────────────────────╢ |
| ║ domain_without_subdomains │ cnn.com │ 'cnn.com' ║ |
| ╟───────────────────────────┼───────────────────────────────────────────┼─────────────────────────────────────╢ |
| ║ upper_domain │ CNN.COM │ TO_UPPER(domain_without_subdomains) ║ |
| ╚═══════════════════════════╧═══════════════════════════════════════════╧═════════════════════════════════════╝ |
| |
| [Stellar]>>> # We can add upper_domain as a transformation to the parser now by |
| [Stellar]>>> ?PARSER_STELLAR_TRANSFORM_ADD |
| PARSER_STELLAR_TRANSFORM_ADD |
| Description: Add stellar field transformation. |
| |
| Arguments: |
| sensorConfig - Sensor config to add transformation to. |
| stellarTransforms - A Map associating fields to stellar expressions |
| |
| Returns: The String representation of the config in zookeeper |
| [Stellar]>>> # We will use the SHELL_VARS2MAP to construct our map associating the field name with the expression |
| [Stellar]>>> ?SHELL_VARS2MAP |
| SHELL_VARS2MAP |
| Description: Take a set of variables and return a map |
| |
| Arguments: |
| variables* - variable names to use to create map |
| |
| Returns: A map associating the variable name with the stellar expression. |
| [Stellar]>>> squid_parser_config_new := PARSER_STELLAR_TRANSFORM_ADD( squid_parser_config, SHELL_VARS2MAP('upper_domain') ) |
| [Stellar]>>> #Now we can make sure that we have the transformation added |
| [Stellar]>>> PARSER_STELLAR_TRANSFORM_PRINT(squid_parser_config_new) |
| ╔═══════════════════════════╤═════════════════════════════════════════╗ |
| ║ Field │ Transformation ║ |
| ╠═══════════════════════════╪═════════════════════════════════════════╣ |
| ║ full_hostname │ URL_TO_HOST(url) ║ |
| ╟───────────────────────────┼─────────────────────────────────────────╢ |
| ║ domain_without_subdomains │ DOMAIN_REMOVE_SUBDOMAINS(full_hostname) ║ |
| ╟───────────────────────────┼─────────────────────────────────────────╢ |
| ║ upper_domain │ TO_UPPER(domain_without_subdomains) ║ |
| ╚═══════════════════════════╧═════════════════════════════════════════╝ |
| |
| [Stellar]>>> #And finally, we push the configs back to zookeeper |
| [Stellar]>>> CONFIG_PUT('PARSER', squid_parser_config_new, 'squid') |
| [Stellar]>>> #Now we can make sure that we have the transformation added into zookeeper |
| [Stellar]>>> PARSER_STELLAR_TRANSFORM_PRINT(CONFIG_GET('PARSER', 'squid')) |
| ╔═══════════════════════════╤═════════════════════════════════════════╗ |
| ║ Field │ Transformation ║ |
| ╠═══════════════════════════╪═════════════════════════════════════════╣ |
| ║ full_hostname │ URL_TO_HOST(url) ║ |
| ╟───────────────────────────┼─────────────────────────────────────────╢ |
| ║ domain_without_subdomains │ DOMAIN_REMOVE_SUBDOMAINS(full_hostname) ║ |
| ╟───────────────────────────┼─────────────────────────────────────────╢ |
| ║ upper_domain │ TO_UPPER(domain_without_subdomains) ║ |
| ╚═══════════════════════════╧═════════════════════════════════════════╝ |
| |
| [Stellar]>>> #Now that we have added it, we can change our mind and remove it |
| [Stellar]>>> ?PARSER_STELLAR_TRANSFORM_REMOVE |
| PARSER_STELLAR_TRANSFORM_REMOVE |
| Description: Remove stellar field transformation. |
| |
| Arguments: |
| sensorConfig - Sensor config to add transformation to. |
| stellarTransforms - A list of stellar transforms to remove |
| |
| Returns: The String representation of the config in zookeeper |
| [Stellar]>>> squid_parser_config_new := PARSER_STELLAR_TRANSFORM_REMOVE( CONFIG_GET('PARSER', 'squid'), [ 'upper_domain' ] ) |
| [Stellar]>>> PARSER_STELLAR_TRANSFORM_PRINT(squid_parser_config_new) |
| ╔═══════════════════════════╤═════════════════════════════════════════╗ |
| ║ Field │ Transformation ║ |
| ╠═══════════════════════════╪═════════════════════════════════════════╣ |
| ║ full_hostname │ URL_TO_HOST(url) ║ |
| ╟───────────────────────────┼─────────────────────────────────────────╢ |
| ║ domain_without_subdomains │ DOMAIN_REMOVE_SUBDOMAINS(full_hostname) ║ |
| ╚═══════════════════════════╧═════════════════════════════════════════╝ |
| |
| [Stellar]>>> #We can now push up the config to zookeeper |
| [Stellar]>>> CONFIG_PUT('PARSER', squid_parser_config_new, 'squid') |
| [Stellar]>>> #It should be just as we started the exercise |
| [Stellar]>>> CONFIG_GET('PARSER', 'squid') |
| { |
| "parserClassName" : "org.apache.metron.parsers.GrokParser", |
| "sensorTopic" : "squid", |
| "parserConfig" : { |
| "grokPath" : "/patterns/squid", |
| "patternLabel" : "SQUID_DELIMITED", |
| "timestampField" : "timestamp" |
| }, |
| "fieldTransformations" : [ { |
| "input" : [ ], |
| "output" : [ "full_hostname", "domain_without_subdomains" ], |
| "transformation" : "STELLAR", |
| "config" : { |
| "full_hostname" : "URL_TO_HOST(url)", |
| "domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" |
| } |
| } ] |
| } |
| [Stellar]>>> #And quit the REPL |
| [Stellar]>>> quit |
| </pre></div></div> |
| </div> |
| <div class="section"> |
| <h3><a name="Manage_Stellar_Enrichments"></a>Manage Stellar Enrichments</h3> |
| |
| <div> |
| <div> |
| <pre class="source">1010 [main] INFO o.a.c.f.i.CuratorFrameworkImpl - Starting |
| 1077 [main-EventThread] INFO o.a.c.f.s.ConnectionStateManager - State change: CONNECTED |
| Stellar, Go! |
| Please note that functions are loading lazily in the background and will be unavailable until loaded fully. |
| {es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH} |
| [Stellar]>>> # First we get the squid enrichment config from zookeeper. |
| [Stellar]>>> # If it is not there, which it is not by default, a suitable default |
| [Stellar]>>> # config will be specified. |
| [Stellar]>>> squid_enrichment_config := CONFIG_GET('ENRICHMENT', 'squid') |
| 26307 [Thread-1] INFO o.r.Reflections - Reflections took 24845 ms to scan 22 urls, producing 17898 keys and 121520 values |
| 26389 [Thread-1] INFO o.a.m.c.d.FunctionResolverSingleton - Found 84 Stellar Functions... |
| Functions loaded, you may refer to functions now... |
| [Stellar]>>> # Just to make sure it looks right, we can view the JSON |
| [Stellar]>>> squid_enrichment_config |
| { |
| "enrichment" : { |
| "fieldMap" : { }, |
| "fieldToTypeMap" : { }, |
| "config" : { } |
| }, |
| "threatIntel" : { |
| "fieldMap" : { }, |
| "fieldToTypeMap" : { }, |
| "config" : { }, |
| "triageConfig" : { |
| "riskLevelRules" : [ ], |
| "aggregator" : "MAX", |
| "aggregationConfig" : { } |
| } |
| }, |
| "configuration" : { } |
| } |
| [Stellar]>>> # Now that we have a config, we can add an enrichment to the Stellar adapter |
| [Stellar]>>> # We should make sure that the current enrichment does not have any already |
| [Stellar]>>> ?ENRICHMENT_STELLAR_TRANSFORM_PRINT |
| ENRICHMENT_STELLAR_TRANSFORM_PRINT |
| Description: Retrieve stellar enrichment transformations. |
| |
| Arguments: |
| sensorConfig - Sensor config to add transformation to. |
| type - ENRICHMENT or THREAT_INTEL |
| |
| Returns: The String representation of the transformations |
| [Stellar]>>> # Since there are two places we can add enrichments, we should check both |
| [Stellar]>>> ENRICHMENT_STELLAR_TRANSFORM_PRINT(squid_enrichment_config, 'ENRICHMENT') |
| ╔═══════╤═══════╤════════════════╗ |
| ║ Group │ Field │ Transformation ║ |
| ╠═══════╧═══════╧════════════════╣ |
| ║ (empty) ║ |
| ╚════════════════════════════════╝ |
| |
| [Stellar]>>> ENRICHMENT_STELLAR_TRANSFORM_PRINT(squid_enrichment_config, 'THREAT_INTEL') |
| ╔═══════╤═══════╤════════════════╗ |
| ║ Group │ Field │ Transformation ║ |
| ╠═══════╧═══════╧════════════════╣ |
| ║ (empty) ║ |
| ╚════════════════════════════════╝ |
| |
| [Stellar]>>> # For my enrichment, I want to add a field indicating that the src address is local or not |
| [Stellar]>>> # I will define local as part of '192.168.0.0/21' |
| [Stellar]>>> # We must be careful about the ip_src_addr, what if it is malformed? |
| [Stellar]>>> ip_src_addr := '200.20.10.1' |
| [Stellar]>>> IN_SUBNET( ip_src_addr, '192.168.0.0/21') |
| false |
| [Stellar]>>> # Just as we expected. Now we can try a local address |
| [Stellar]>>> ip_src_addr := '192.168.0.1' |
| [Stellar]>>> IN_SUBNET( ip_src_addr, '192.168.0.0/21') |
| true |
| [Stellar]>>> # Just as we expected. Now we can try some malformed ones |
| [Stellar]>>> ip_src_addr := NULL |
| [Stellar]>>> IN_SUBNET( ip_src_addr, '192.168.0.0/21') |
| false |
| [Stellar]>>> # So far, so good |
| [Stellar]>>> ip_src_addr := 'foo bar' |
| [Stellar]>>> IN_SUBNET( ip_src_addr, '192.168.0.0/21') |
| [!] Unable to execute: Could not parse [foo bar] |
| [Stellar]>>> # uh oh, that was terrible, we will have to adjust and be a bit more defensive |
| [Stellar]>>> IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21') |
| false |
| [Stellar]>>> ip_src_addr := NULL |
| [Stellar]>>> IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21') |
| false |
| [Stellar]>>> ip_src_addr := '192.168.0.1' |
| [Stellar]>>> IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21') |
| true |
| [Stellar]>>> ip_src_addr := '200.20.10.1' |
| [Stellar]>>> IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21') |
| false |
| [Stellar]>>> # I think we are ready, we will call it is_local |
| [Stellar]>>> is_local := IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21') |
| [Stellar]>>> # Now we can add the function to the ENRICHMENT phase of the enrichment topology |
| [Stellar]>>> ?ENRICHMENT_STELLAR_TRANSFORM_ADD |
| ENRICHMENT_STELLAR_TRANSFORM_ADD |
| Description: Add stellar field transformation. |
| |
| Arguments: |
| sensorConfig - Sensor config to add transformation to. |
| type - ENRICHMENT or THREAT_INTEL |
| stellarTransforms - A Map associating fields to stellar expressions |
| group - Group to add to (optional) |
| |
| Returns: The String representation of the config in zookeeper |
| [Stellar]>>> squid_enrichment_config_new := ENRICHMENT_STELLAR_TRANSFORM_ADD( squid_enrichment_config, 'ENRICHMENT', SHELL_VARS2MAP( 'is_local' ) ) |
| [Stellar]>>> # Make sure that it is really there |
| [Stellar]>>> ENRICHMENT_STELLAR_TRANSFORM_PRINT(squid_enrichment_config_new, 'ENRICHMENT') |
| ╔═══════════╤══════════╤════════════════════════════════════════════════════════════════════════════════╗ |
| ║ Group │ Field │ Transformation ║ |
| ╠═══════════╪══════════╪════════════════════════════════════════════════════════════════════════════════╣ |
| ║ (default) │ is_local │ IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21') ║ |
| ╚═══════════╧══════════╧════════════════════════════════════════════════════════════════════════════════╝ |
| |
| [Stellar]>>> # and not in threat intel |
| [Stellar]>>> ENRICHMENT_STELLAR_TRANSFORM_PRINT(squid_enrichment_config_new, 'THREAT_INTEL') |
| ╔═══════╤═══════╤════════════════╗ |
| ║ Group │ Field │ Transformation ║ |
| ╠═══════╧═══════╧════════════════╣ |
| ║ (empty) ║ |
| ╚════════════════════════════════╝ |
| |
| [Stellar]>>> # And see it in the JSON |
| [Stellar]>>> squid_enrichment_config_new |
| { |
| "index" : "squid", |
| "batchSize" : 100, |
| "enrichment" : { |
| "fieldMap" : { |
| "stellar" : { |
| "config" : { |
| "is_local" : "IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21')" |
| } |
| } |
| }, |
| "fieldToTypeMap" : { }, |
| "config" : { } |
| }, |
| "threatIntel" : { |
| "fieldMap" : { }, |
| "fieldToTypeMap" : { }, |
| "config" : { }, |
| "triageConfig" : { |
| "riskLevelRules" : [ ], |
| "aggregator" : "MAX", |
| "aggregationConfig" : { } |
| } |
| }, |
| "configuration" : { } |
| } |
| [Stellar]>>> # Now we can push the configs to zookeeper |
| [Stellar]>>> CONFIG_PUT( 'ENRICHMENT', squid_enrichment_config_new, 'squid') |
| [Stellar]>>> # And validate that our JSON looks a lot better |
| [Stellar]>>> CONFIG_GET( 'ENRICHMENT', 'squid') |
| { |
| "index" : "squid", |
| "batchSize" : 100, |
| "enrichment" : { |
| "fieldMap" : { |
| "stellar" : { |
| "config" : { |
| "is_local" : "IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21')" |
| } |
| } |
| }, |
| "fieldToTypeMap" : { }, |
| "config" : { } |
| }, |
| "threatIntel" : { |
| "fieldMap" : { }, |
| "fieldToTypeMap" : { }, |
| "config" : { }, |
| "triageConfig" : { |
| "riskLevelRules" : [ ], |
| "aggregator" : "MAX", |
| "aggregationConfig" : { } |
| } |
| }, |
| "configuration" : { } |
| } |
| [Stellar]>>> # You know what? I changed my mind, I do not want that enrichment, so we can remove it pretty easily. |
| [Stellar]>>> ?ENRICHMENT_STELLAR_TRANSFORM_REMOVE |
| ENRICHMENT_STELLAR_TRANSFORM_REMOVE |
| Description: Remove one or more stellar field transformations. |
| |
| Arguments: |
| sensorConfig - Sensor config to add transformation to. |
| type - ENRICHMENT or THREAT_INTEL |
| stellarTransforms - A list of removals |
| group - Group to remove from (optional) |
| |
| Returns: The String representation of the config in zookeeper |
| [Stellar]>>> squid_enrichment_config_new := ENRICHMENT_STELLAR_TRANSFORM_REMOVE( squid_enrichment_config_new, 'ENRICHMENT', [ 'is_local' ] ) |
| [Stellar]>>> # Make sure that it is really gone |
| [Stellar]>>> ENRICHMENT_STELLAR_TRANSFORM_PRINT(squid_enrichment_config_new, 'ENRICHMENT') |
| ╔═══════╤═══════╤════════════════╗ |
| ║ Group │ Field │ Transformation ║ |
| ╠═══════╧═══════╧════════════════╣ |
| ║ (empty) ║ |
| ╚════════════════════════════════╝ |
| |
| [Stellar]>>> # and not in threat intel |
| [Stellar]>>> ENRICHMENT_STELLAR_TRANSFORM_PRINT(squid_enrichment_config_new, 'THREAT_INTEL') |
| ╔═══════╤═══════╤════════════════╗ |
| ║ Group │ Field │ Transformation ║ |
| ╠═══════╧═══════╧════════════════╣ |
| ║ (empty) ║ |
| ╚════════════════════════════════╝ |
| |
| [Stellar]>>> # Ok, I feel comfortable updating now |
| [Stellar]>>> CONFIG_PUT( 'ENRICHMENT', squid_enrichment_config_new, 'squid') |
| [Stellar]>>> CONFIG_GET( 'ENRICHMENT', 'squid') |
| { |
| "index" : "squid", |
| "batchSize" : 100, |
| "enrichment" : { |
| "fieldMap" : { }, |
| "fieldToTypeMap" : { }, |
| "config" : { } |
| }, |
| "threatIntel" : { |
| "fieldMap" : { }, |
| "fieldToTypeMap" : { }, |
| "config" : { }, |
| "triageConfig" : { |
| "riskLevelRules" : [ ], |
| "aggregator" : "MAX", |
| "aggregationConfig" : { } |
| } |
| }, |
| "configuration" : { } |
| } |
| [Stellar]>>> |
| </pre></div></div> |
| </div> |
| <div class="section"> |
| <h3><a name="Manage_Threat_Triage_Rules"></a>Manage Threat Triage Rules</h3> |
| |
| <div> |
| <div> |
| <pre class="source">987 [main] INFO o.a.c.f.i.CuratorFrameworkImpl - Starting |
| 1047 [main-EventThread] INFO o.a.c.f.s.ConnectionStateManager - State change: CONNECTED |
| Stellar, Go! |
| Please note that functions are loading lazily in the background and will be unavailable until loaded fully. |
| {es.clustername=metron, es.ip=node1, es.port=9300, es.date.format=yyyy.MM.dd.HH} |
| [Stellar]>>> # First we get the squid enrichment config from zookeeper. |
| [Stellar]>>> # If it is not there, which it is not by default, a suitable default |
| [Stellar]>>> # config will be specified. |
| [Stellar]>>> squid_enrichment_config := CONFIG_GET('ENRICHMENT', 'squid') |
| 26751 [Thread-1] INFO o.r.Reflections - Reflections took 24407 ms to scan 22 urls, producing 17898 keys and 121520 values |
| 26828 [Thread-1] INFO o.a.m.c.d.FunctionResolverSingleton - Found 84 Stellar Functions... |
| Functions loaded, you may refer to functions now... |
| [Stellar]>>> # We should not have any threat triage rules |
| [Stellar]>>> THREAT_TRIAGE_PRINT(squid_enrichment_config) |
| ╔═════════════╤═══════╗ |
| ║ Triage Rule │ Score ║ |
| ╠═════════════╧═══════╣ |
| ║ (empty) ║ |
| ╚═════════════════════╝ |
| |
| [Stellar]>>> # I have followed the blog post at https://cwiki.apache.org/confluence/display/METRON/2016/04/28/Metron+Tutorial+-+Fundamentals+Part+2%3A+Creating+a+New+Enrichment |
| [Stellar]>>> # and have some enrichment reference data loaded into hbase, |
| [Stellar]>>> # so we should be able to retrieve that as an enrichment using the ENRICHMENT_GET |
| [Stellar]>>> # function to call out to hbase from stellar |
| [Stellar]>>> ?ENRICHMENT_GET |
| ENRICHMENT_GET |
| Description: Interrogates the HBase table holding the simple hbase enrichment data and retrieves the tabular value associated with the enrichment type and indicator. |
| |
| Arguments: |
| enrichment_type - The enrichment type |
| indicator - The string indicator to look up |
| nosql_table - The NoSQL Table to use |
| column_family - The Column Family to use |
| |
| Returns: A Map associated with the indicator and enrichment type. Empty otherwise. |
| [Stellar]>>> domain_without_subdomains := 'cnn.com' |
| [Stellar]>>> whois_info := ENRICHMENT_GET('whois', domain_without_subdomains, 'enrichment', 't') |
| [Stellar]>>> # Now I know that the whois_info is actually a map of values, so we can use the SHELL_MAP2TABLE function |
| [Stellar]>>> # to get a nicer view on this data |
| [Stellar]>>> SHELL_MAP2TABLE(whois_info) |
| ╔══════════════════════════╤══════════════════════════════════╗ |
| ║ KEY │ VALUE ║ |
| ╠══════════════════════════╪══════════════════════════════════╣ |
| ║ owner │ Turner Broadcasting System, Inc. ║ |
| ╟──────────────────────────┼──────────────────────────────────╢ |
| ║ registrar │ Domain Name Manager ║ |
| ╟──────────────────────────┼──────────────────────────────────╢ |
| ║ domain_created_timestamp │ 748695600000 ║ |
| ╟──────────────────────────┼──────────────────────────────────╢ |
| ║ home_country │ US ║ |
| ╟──────────────────────────┼──────────────────────────────────╢ |
| ║ domain │ cnn.com ║ |
| ╚══════════════════════════╧══════════════════════════════════╝ |
| |
| [Stellar]>>> #Looks good |
| [Stellar]>>> squid_enrichment_config_new := ENRICHMENT_STELLAR_TRANSFORM_ADD( squid_enrichment_config, 'ENRICHMENT', SHELL_VARS2MAP( 'whois_info' ) ) |
| [Stellar]>>> # Just for illustration, we can create a threat alert if the country of the domain registered |
| [Stellar]>>> # is non-US, then we can make an alert. To do that, we need to create an is_alert field on the message. |
| [Stellar]>>> # |
| [Stellar]>>> # I know that maps get folded into the message, so that whois_info enrichment is going to create a few fields: |
| [Stellar]>>> # * domain mapped to whois_info.domain |
| [Stellar]>>> # * registrar mapped to whois_info.registrar |
| [Stellar]>>> # * home_country mapped to whois_info.home_country |
| [Stellar]>>> # * owner mapped to whois_info.owner |
| [Stellar]>>> whois_info.home_country := 'US' |
| [Stellar]>>> is_alert := whois_info.home_country != 'US' |
| [Stellar]>>> # Now we can add this Stellar enrichment to the THREAT_INTEL portion of the enrichment topology |
| [Stellar]>>> squid_enrichment_config_new := ENRICHMENT_STELLAR_TRANSFORM_ADD( squid_enrichment_config_new, 'THREAT_INTEL', SHELL_VARS2MAP( 'is_alert' ) ) |
| [Stellar]>>> # We should recap to make sure we know what enrichments we have |
| [Stellar]>>> ENRICHMENT_STELLAR_TRANSFORM_PRINT(squid_enrichment_config_new, 'ENRICHMENT') |
| ╔═══════════╤════════════╤═══════════════════════════════════════════════════════════════════════╗ |
| ║ Group │ Field │ Transformation ║ |
| ╠═══════════╪════════════╪═══════════════════════════════════════════════════════════════════════╣ |
| ║ (default) │ whois_info │ ENRICHMENT_GET('whois', domain_without_subdomains, 'enrichment', 't') ║ |
| ╚═══════════╧════════════╧═══════════════════════════════════════════════════════════════════════╝ |
| |
| [Stellar]>>> ENRICHMENT_STELLAR_TRANSFORM_PRINT(squid_enrichment_config_new, 'THREAT_INTEL') |
| ╔═══════════╤══════════╤═════════════════════════════════╗ |
| ║ Group │ Field │ Transformation ║ |
| ╠═══════════╪══════════╪═════════════════════════════════╣ |
| ║ (default) │ is_alert │ whois_info.home_country != 'US' ║ |
| ╚═══════════╧══════════╧═════════════════════════════════╝ |
| |
| [Stellar]>>> # Now with this, we can create a rule or two to triage these alerts. |
| [Stellar]>>> # This means associating a rule as described by a stellar expression that returns true or false with a score |
| [Stellar]>>> # Also associated with this ruleset is an aggregation function, the default of which is MAX. |
| [Stellar]>>> # Now we can make a couple rules: |
| [Stellar]>>> # * If the message is an alert and from a non-us whois source, we can set the level to 10 |
| [Stellar]>>> # * If the message is an alert and non-local, we can set the level to 20 |
| [Stellar]>>> # * If the message is an alert and both non-local and non-us, then we can set the level to 50 |
| [Stellar]>>> # If multiple rules hit, then we should take the max (default behavior) |
| [Stellar]>>> non_us := whois_info.home_country != 'US' |
| [Stellar]>>> is_local := IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21') |
| [Stellar]>>> is_both := whois_info.home_country != 'US' && IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21') |
| [Stellar]>>> rules := [ { 'name' : 'is non-us', 'rule' : SHELL_GET_EXPRESSION('non_us'), 'score' : 10 } , { 'name' : 'is local', 'rule' : SHELL_GET_EXPRESSION('is_local'), 'score' : 20 } , { 'name' : 'both non-us and local', 'comment' : 'union of both rules.', 'rule' : SHELL_GET_EXPRESSION('is_both'), 'score' : 50 } ] |
| [Stellar]>>> # Now that we have our rules staged, we can add them to our config. |
| [Stellar]>>> squid_enrichment_config_new := THREAT_TRIAGE_ADD( squid_enrichment_config_new, rules ) |
| [Stellar]>>> THREAT_TRIAGE_PRINT(squid_enrichment_config_new) |
| ╔═══════════════════════╤══════════════════════╤═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════╤═══════╗ |
| ║ Name │ Comment │ Triage Rule │ Score ║ |
| ╠═══════════════════════╪══════════════════════╪═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════╪═══════╣ |
| ║ is non-us │ │ whois_info.home_country != 'US' │ 10 ║ |
| ╟───────────────────────┼──────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┼───────╢ |
| ║ is local │ │ IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21') │ 20 ║ |
| ╟───────────────────────┼──────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┼───────╢ |
| ║ both non-us and local │ union of both rules. │ whois_info.home_country != 'US' && IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21') │ 50 ║ |
| ╚═══════════════════════╧══════════════════════╧═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════╧═══════╝ |
| |
| Aggregation: MAX |
| [Stellar]>>> # Looks good, we can push the configs up |
| [Stellar]>>> CONFIG_PUT('ENRICHMENT', squid_enrichment_config_new, 'squid') |
| [Stellar]>>> # And admire the resulting JSON that you did not have to edit directly. |
| [Stellar]>>> CONFIG_GET('ENRICHMENT', 'squid') |
| { |
| "enrichment" : { |
| "fieldMap" : { |
| "stellar" : { |
| "config" : { |
| "whois_info" : "ENRICHMENT_GET('whois', domain_without_subdomains, 'enrichment', 't')" |
| } |
| } |
| }, |
| "fieldToTypeMap" : { }, |
| "config" : { } |
| }, |
| "threatIntel" : { |
| "fieldMap" : { |
| "stellar" : { |
| "config" : { |
| "is_alert" : "whois_info.home_country != 'US'" |
| } |
| } |
| }, |
| "fieldToTypeMap" : { }, |
| "config" : { }, |
| "triageConfig" : { |
| "riskLevelRules" : [ { |
| "name" : "is non-us", |
| "rule" : "whois_info.home_country != 'US'", |
| "score" : 10.0 |
| }, { |
| "name" : "is local", |
| "rule" : "IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21')", |
| "score" : 20.0 |
| }, { |
| "name" : "both non-us and local", |
| "comment" : "union of both rules.", |
| "rule" : "whois_info.home_country != 'US' && IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21')", |
| "score" : 50.0 |
| } ], |
| "aggregator" : "MAX", |
| "aggregationConfig" : { } |
| } |
| }, |
| "configuration" : { } |
| } |
| [Stellar]>>> # Now that we have admired it, we can remove the rules |
| [Stellar]>>> squid_enrichment_config_new := THREAT_TRIAGE_REMOVE( squid_enrichment_config_new, [ SHELL_GET_EXPRESSION('non_us') , SHELL_GET_EXPRESSION('is_local') , SHELL_GET_EXPRES |
| SION('is_both') ] ) |
| [Stellar]>>> THREAT_TRIAGE_PRINT(squid_enrichment_config_new) |
| ╔══════╤═════════╤═════════════╤═══════╗ |
| ║ Name │ Comment │ Triage Rule │ Score ║ |
| ╠══════╧═════════╧═════════════╧═══════╣ |
| ║ (empty) ║ |
| ╚══════════════════════════════════════╝ |
| |
| [Stellar]>>> # and push configs |
| [Stellar]>>> CONFIG_PUT('ENRICHMENT', squid_enrichment_config_new, 'squid') |
| [Stellar]>>> # And admire the resulting JSON that is devoid of threat triage rules. |
| [Stellar]>>> CONFIG_GET('ENRICHMENT', 'squid') |
| { |
| "enrichment" : { |
| "fieldMap" : { |
| "stellar" : { |
| "config" : { |
| "whois_info" : "ENRICHMENT_GET('whois', domain_without_subdomains, 'enrichment', 't')" |
| } |
| } |
| }, |
| "fieldToTypeMap" : { }, |
| "config" : { } |
| }, |
| "threatIntel" : { |
| "fieldMap" : { |
| "stellar" : { |
| "config" : { |
| "is_alert" : "whois_info.home_country != 'US'" |
| } |
| } |
| }, |
| "fieldToTypeMap" : { }, |
| "config" : { }, |
| "triageConfig" : { |
| "riskLevelRules" : [ ], |
| "aggregator" : "MAX", |
| "aggregationConfig" : { } |
| } |
| }, |
| "configuration" : { } |
| } |
| [Stellar]>>> |
| </pre></div></div></div></div> |
| </div> |
| </div> |
| </div> |
| <hr/> |
| <footer> |
| <div class="container-fluid"> |
| <div class="row-fluid"> |
| © 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo, |
| and the Apache Metron project logo are trademarks of The Apache Software Foundation. |
| </div> |
| </div> |
| </footer> |
| </body> |
| </html> |
