eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml

<?xml version="1.0" encoding="UTF-8" ?>
<!--
  ~ Licensed to the Apache Software Foundation (ASF) under one or more
  ~ contributor license agreements.  See the NOTICE file distributed with
  ~ this work for additional information regarding copyright ownership.
  ~ The ASF licenses this file to You under the Apache License, Version 2.0
  ~ (the "License"); you may not use this file except in compliance with
  ~ the License.  You may obtain a copy of the License at
  ~
  ~    http://www.apache.org/licenses/LICENSE-2.0
  ~
  ~ Unless required by applicable law or agreed to in writing, software
  ~ distributed under the License is distributed on an "AS IS" BASIS,
  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  -->

<application>
    <type>OOZIE_AUDIT_LOG_MONITOR</type>
    <name>Oozie Audit Log Monitor</name>
    <appClass>org.apache.eagle.security.oozie.parse.OozieAuditLogApplication</appClass>
    <configuration>
        <!-- topology related configurations -->
        <property>
            <name>workers</name>
            <displayName>storm workers</displayName>
            <value>1</value>
            <description>number of topology workers</description>
        </property>
        <property>
            <name>topology.numOfJoinTasks</name>
            <displayName>Topology Join Tasks</displayName>
            <value>2</value>
            <description>number of join tasks</description>
        </property>
        <property>
            <name>topology.numOfSpoutTasks</name>
            <displayName>Topology Spout Tasks</displayName>
            <value>2</value>
            <description>number of spout tasks</description>
        </property>
        <property>
            <name>topology.numOfParserTasks</name>
            <displayName>Topology Parser Tasks</displayName>
            <value>2</value>
            <description>number of parser tasks</description>
        </property>
        <property>
            <name>topology.numOfSensitivityJoinTasks</name>
            <displayName>Topology Sensitivity JoinTasks</displayName>
            <value>2</value>
            <description>number of sensitivity join tasks</description>
        </property>
        <property>
            <name>topology.numOfIPZoneJoinTasks</name>
            <displayName>Topology IPZone JoinTasks</displayName>
            <value>2</value>
            <description>number of ip zone join tasks</description>
        </property>
        <property>
            <name>topology.numOfSinkTasks</name>
            <displayName>Topology Sink Tasks</displayName>
            <value>2</value>
            <description>number of sink tasks</description>
        </property>

        <!-- data source configurations -->
        <property>
            <name>dataSourceConfig.topic</name>
            <displayName>Kafka Topic for Data Consumption</displayName>
            <value>oozie_audit_log_${siteId}</value>
            <description>data source topic</description>
        </property>
        <property>
            <name>dataSourceConfig.zkConnection</name>
            <displayName>Kafka Zookeeper Connection</displayName>
            <value>localhost:2181</value>
            <description>kafka broker zk connection</description>
            <required>true</required>
        </property>
        <property>
            <name>dataSourceConfig.txZkServers</name>
            <displayName>Kafka Zookeeper servers</displayName>
            <value>localhost:2181</value>
            <description>zookeeper server for offset transaction</description>
        </property>
        <property>
            <name>dataSourceConfig.schemeCls</name>
            <displayName>Kafka Consumer SchemeCls</displayName>
            <value>storm.kafka.StringScheme</value>
            <description>scheme class</description>
        </property>

        <!-- data enrich configurations -->
        <property>
            <name>dataEnrich.dataJoinPollIntervalSec</name>
            <displayName>Data Join Poll Interval Sec</displayName>
            <value>30</value>
            <description>interval in seconds for polling</description>
        </property>

        <!-- eagle service configurations -->
        <property>
            <name>eagleService.host</name>
            <displayName>Eagle Service Host</displayName>
            <value>localhost</value>
            <description>eagle service host</description>
        </property>
        <property>
            <name>eagleService.port</name>
            <displayName>Eagle Service Port</displayName>
            <value>9090</value>
            <description>eagle service port</description>
        </property>
        <property>
            <name>eagleService.username</name>
            <displayName>Eagle Service Username</displayName>
            <value>admin</value>
            <description>eagle service username</description>
        </property>
        <property>
            <name>eagleService.password</name>
            <displayName>Eagle Service Password</displayName>
            <value>secret</value>
            <description>eagle service password</description>
        </property>

        <!-- data sink configurations -->
        <property>
            <name>dataSinkConfig.topic</name>
            <displayName>Kafka Topic for Parsed Data Sink</displayName>
            <value>oozie_audit_event_${siteId}</value>
            <description>topic for kafka data sink</description>
        </property>
        <property>
            <name>dataSinkConfig.brokerList</name>
            <displayName>Kafka BrokerList for Data Sink</displayName>
            <value>localhost:6667</value>
            <description>kafka broker list</description>
        </property>
        <property>
            <name>dataSinkConfig.serializerClass</name>
            <displayName>Kafka Producer SerializerClass</displayName>
            <value>kafka.serializer.StringEncoder</value>
            <description>serializer class Kafka message value</description>
        </property>
        <property>
            <name>dataSinkConfig.keySerializerClass</name>
            <displayName>Kafka Producer keySerializerClass</displayName>
            <value>kafka.serializer.StringEncoder</value>
            <description>serializer class Kafka message key</description>
        </property>

        <property>
            <name>dataSinkConfig.producerType</name>
            <displayName>Kafka Producer Type</displayName>
            <value>async</value>
            <description>whether the messages are sent asynchronously in a background thread</description>
        </property>
        <property>
            <name>dataSinkConfig.numBatchMessages</name>
            <displayName>Kafka Producer NumBatchMessages</displayName>
            <value>4096</value>
            <description>number of messages to send in one batch when using async mode</description>
        </property>
        <property>
            <name>dataSinkConfig.maxQueueBufferMs</name>
            <displayName>Kafka Producer MaxQueueBufferMs</displayName>
            <value>5000</value>
            <description>maximum time to buffer data when using async mode</description>
        </property>
        <property>
            <name>dataSinkConfig.requestRequiredAcks</name>
            <displayName>Kafka Producer RequestRequiredAcks</displayName>
            <value>0</value>
            <description>value controls when a produce request is considered completed</description>
        </property>
    </configuration>
    <streams>
        <stream>
            <streamId>oozie_audit_log_stream</streamId>
            <description>oozie Audit Log Stream</description>
            <validate>true</validate>
            <timeseries>true</timeseries>
            <columns>
                <column>
                    <name>user</name>
                    <type>string</type>
                </column>
                <column>
                    <name>ip</name>
                    <type>string</type>
                </column>
                <column>
                    <name>jobId</name>
                    <type>string</type>
                </column>
                <column>
                    <name>operation</name>
                    <type>string</type>
                </column>
                <column>
                    <name>timestamp</name>
                    <type>long</type>
                </column>
            </columns>
        </stream>
    </streams>
    <docs>
        <install>
            # Step 1: Create source kafka topic named "${site}_example_source_topic"

            ./bin/kafka-topics.sh --create --topic example_source_topic --replication-factor 1 --replication 1

            # Step 2: Set up data collector to flow data into kafka topic in

            ./bin/logstash -f log_collector.conf

            ## `log_collector.conf` sample as following:

            input {

            }
            filter {

            }
            output{

            }

            # Step 3: start application

            # Step 4: monitor with featured portal or alert with policies
        </install>
        <uninstall>
            # Step 1: stop and uninstall application
            # Step 2: delete kafka topic named "${site}_example_source_topic"
            # Step 3: stop logstash
        </uninstall>
    </docs>
</application>