ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.ranger.policyengine;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
import org.apache.ranger.plugin.util.PerfDataRecorder;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

public class RangerPolicyenginePerfTester {
    static final Log LOG = LogFactory.getLog(RangerPolicyenginePerfTester.class);

    public static void main(String[] args) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("==> RangerPolicyenginePerfTester.main()");
        }

        CommandLineParser commandLineParser = new CommandLineParser();

        PerfTestOptions perfTestOptions = commandLineParser.parse(args);

        if (perfTestOptions != null) {
            URL statCollectionFileURL = perfTestOptions.getStatCollectionFileURL();

            List<String> perfModuleNames = statCollectionFileURL != null ? buildPerfModuleNames(statCollectionFileURL) : new ArrayList<String>();

            PerfDataRecorder.initialize(perfModuleNames);

            URL servicePoliciesFileURL = perfTestOptions.getServicePoliciesFileURL();

            RangerPolicyEngineOptions policyEngineOptions = new RangerPolicyEngineOptions();
            policyEngineOptions.disableTagPolicyEvaluation = false;
            policyEngineOptions.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED;
            policyEngineOptions.cacheAuditResults = false;
            policyEngineOptions.disableTrieLookupPrefilter = perfTestOptions.getIsTrieLookupPrefixDisabled();
            policyEngineOptions.optimizeTrieForRetrieval = perfTestOptions.getIsOnDemandTriePostSetupDisabled();

            URL configurationFileURL = perfTestOptions.getPerfConfigurationFileURL();

            if (configurationFileURL != null) {
	            RangerConfiguration config = RangerConfiguration.getInstance();
	            config.addResource(configurationFileURL);
            }

            PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, policyEngineOptions, perfTestOptions.getIsDynamicReorderingDisabled());
            if (!perfTestEngine.init()) {
                LOG.error("Error initializing test data. Existing...");
                System.exit(1);
            }

            URL[] requestFileURLs = perfTestOptions.getRequestFileURLs();
            int requestFilesCount = requestFileURLs.length;

            // warm-up policy engine
            LOG.error("Warming up..");
            try {
                for(URL requestFileURL : requestFileURLs) {
                    PerfTestClient perfTestClient = new PerfTestClient(perfTestEngine, 0, requestFileURL, 1);

                    if (perfTestClient.init()) {
                        perfTestClient.start();
                        perfTestClient.join();
                    } else {
                        LOG.error("Error initializing warm-up PerfTestClient");
                    }
                }
            } catch(Throwable t) {
                LOG.error("Error during warmup", t);
            }
            LOG.error("Warmed up!");

            PerfDataRecorder.clearStatistics();

            int clientsCount = perfTestOptions.getConcurrentClientCount();
            List<PerfTestClient> perfTestClients = new ArrayList<PerfTestClient>(clientsCount);

            for (int i = 0; i < clientsCount; i++) {

                URL requestFileURL = requestFileURLs[i % requestFilesCount];

                PerfTestClient perfTestClient = new PerfTestClient(perfTestEngine, i, requestFileURL, perfTestOptions.getIterationsCount());

                if (!perfTestClient.init()) {
                    LOG.error("Error initializing PerfTestClient: (id=" + i + ")");
                } else {
                    perfTestClients.add(perfTestClient);
                }
            }

            if (LOG.isDebugEnabled()) {
                LOG.debug("Number of perfTestClients=" + perfTestClients.size());
            }

            Runtime runtime = Runtime.getRuntime();
            runtime.gc();

            long totalMemory = runtime.totalMemory();
            long freeMemory = runtime.freeMemory();

            LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory);

            LOG.info("Starting " + perfTestClients.size() + " clients..");
            for (PerfTestClient client : perfTestClients) {
                try {
                    client.start();
                } catch (Throwable t) {
                    LOG.error("Error in starting client: " + client.getName(), t);
                }
            }
            LOG.info("Started " + perfTestClients.size() + " clients");

            LOG.info("Waiting for " + perfTestClients.size() + " clients to finish up");

            for (PerfTestClient client : perfTestClients) {
                while (client.isAlive()) {
                    try {
                        client.join(1000);

                        runtime.gc();

                        totalMemory = runtime.totalMemory();
                        freeMemory = runtime.freeMemory();

                        LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory);

                    } catch (InterruptedException interruptedException) {
                        LOG.error("PerfTestClient.join() was interrupted");
                    }
                }
            }

            if (LOG.isDebugEnabled()) {
                LOG.debug("<== RangerPolicyenginePerfTester.main()");
            }

            LOG.info("Completed performance-run");

            perfTestEngine.cleanup();

            PerfDataRecorder.printStatistics();
        }

        LOG.info("Exiting...");

    }

    private static List<String> buildPerfModuleNames(URL statCollectionFileURL) {
        List<String> perfModuleNames = new ArrayList<String>();

        try (
                InputStream inStream = statCollectionFileURL.openStream();
                InputStreamReader reader = new InputStreamReader(inStream, Charset.forName("UTF-8"));
                BufferedReader br = new BufferedReader(reader)
        ) {

            String line;

            while ((line = br.readLine()) != null) {
                line = line.trim();
                if (!line.isEmpty() && !line.startsWith("#")) {
                    String[] moduleNames = line.split(" ");
                    perfModuleNames.addAll(Arrays.asList(moduleNames));
                }
            }
        } catch (IOException exception) {
            System.out.println("Error reading arguments:" + exception);
        }

        return perfModuleNames;
    }
}