metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.metron.solr.integration;

import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.SearchIntegrationTest;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.solr.client.SolrClientFactory;
import org.apache.metron.solr.dao.SolrDao;
import org.apache.metron.solr.integration.components.SolrComponent;
import org.apache.solr.client.solrj.SolrServerException;
import org.json.simple.JSONArray;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

public class SolrSearchIntegrationTest extends SearchIntegrationTest {
  private static SolrComponent solrComponent;
  private static IndexDao dao;

  @BeforeClass
  public static void setupClass() throws Exception {
    indexComponent = startIndex();
    dao = createDao();
    // The data is all static for searches, so we can set it up here, and not do anything between tests.
    broData = SearchIntegrationTest.broData.replace("source:type", "source.type");
    snortData = SearchIntegrationTest.snortData.replace("source:type", "source.type");
    solrComponent.addCollection("bro", "./src/main/config/schema/bro");
    solrComponent.addCollection("snort", "./src/main/config/schema/snort");
    loadTestData();
  }

  @AfterClass
  public static void teardown() {
    SolrClientFactory.close();
    if (solrComponent != null) {
      solrComponent.stop();
    }
  }

  @Override
  public IndexDao getIndexDao() {
    return dao;
  }

  protected static IndexDao createDao() {
    AccessConfig config = new AccessConfig();
    config.setMaxSearchResults(100);
    config.setMaxSearchGroups(100);
    config.setGlobalConfigSupplier( () ->
        new HashMap<String, Object>() {{
          put(SOLR_ZOOKEEPER, solrComponent.getZookeeperUrl());
        }}
    );

    config.setIndexSupplier( sensorType -> sensorType);
    IndexDao dao = new SolrDao();
    dao.init(config);
    return dao;
  }

  protected static InMemoryComponent startIndex() throws Exception {
    solrComponent = new SolrComponent.Builder().build();
    solrComponent.start();
    return solrComponent;
  }

  @SuppressWarnings("unchecked")
  protected static void loadTestData() throws ParseException, IOException, SolrServerException {
    JSONArray broArray = (JSONArray) new JSONParser().parse(broData);
    solrComponent.addDocs("bro", broArray);
    JSONArray snortArray = (JSONArray) new JSONParser().parse(snortData);
    solrComponent.addDocs("snort", snortArray);
  }

  @Override
  @Test
  public void returns_column_metadata_for_specified_indices() throws Exception {
    // getColumnMetadata with only bro
    {
      Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro"));
      // Don't test all fields, just test a sample of different fields
      Assert.assertEquals(263, fieldTypes.size());

      // Fields present in both with same type
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type"));
      Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
      Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));

      // Bro only field
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("username"));

      // A dynamic field present in both with same type
      Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score"));

      // Dyanamic field present in both with nonstandard types.
      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));

      // Field with nonstandard type
      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));

      // Bro only field in the dynamic catch all
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("method"));

      // A field is in both bro and snort and they have different types.
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl"));

      // Field only present in Snort
      Assert.assertEquals(null, fieldTypes.get("dgmlen"));

      // Field that doesn't exist
      Assert.assertEquals(null, fieldTypes.get("fake.field"));
    }
    // getColumnMetadata with only snort
    {
      Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort"));
      Assert.assertEquals(33, fieldTypes.size());

      // Fields present in both with same type
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type"));
      Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
      Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));

      // Snort only field
      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("dgmlen"));

      // A dynamic field present in both with same type
      Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score"));

      // Dyanamic field present in both with nonstandard types.
      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));

      // Field with nonstandard type
      Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));

      // Snort only field in the dynamic catch all
      Assert.assertEquals(FieldType.TEXT, fieldTypes.get("sig_generator"));

      // A field is in both bro and snort and they have different types.
      Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl"));

      // Field only present in Bro
      Assert.assertEquals(null, fieldTypes.get("username"));

      // Field that doesn't exist
      Assert.assertEquals(null, fieldTypes.get("fake.field"));
    }
  }

  @Override
  @Test
  public void returns_column_data_for_multiple_indices() throws Exception {
    Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort"));
    // Don't test everything, just test a variety of fields, including fields across collections.

    // Fields present in both with same type
    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid"));
    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type"));
    Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
    Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
    Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));

    // Bro only field
    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("username"));

    // Snort only field
    Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("dgmlen"));

    // A dynamic field present in both with same type
    Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score"));

    // Dyanamic field present in both with nonstandard types.
    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));

    // Field present in both with nonstandard type
    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp"));

    // Bro only field in the dynamic catch all
    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("method"));

    // Snort only field in the dynamic catch all
    Assert.assertEquals(FieldType.TEXT, fieldTypes.get("sig_generator"));

    // A field is in both bro and snort and they have different types.
    Assert.assertEquals(FieldType.OTHER, fieldTypes.get("ttl"));

    // Field that doesn't exist
    Assert.assertEquals(null, fieldTypes.get("fake.field"));
  }

  @Test
  public void different_type_filter_query() throws Exception {
    thrown.expect(InvalidSearchException.class);
    SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFilterQuery, SearchRequest.class);
    SearchResponse response = dao.search(request);
  }

  @Override
  protected String getSourceTypeField() {
    return Constants.SENSOR_TYPE;
  }

  @Override
  protected String getIndexName(String sensorType) {
    return sensorType;
  }
}