metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.metron.indexing.dao;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.GroupResult;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.integration.InMemoryComponent;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;

public abstract class SearchIntegrationTest {
  /**
   * [
   * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "method": "bro data 1", "ttl": "data 1", "guid":"bro_1"},
   * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "method": "bro data 2", "ttl": "data 2", "guid":"bro_2"},
   * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "method": "bro data 3", "ttl": "data 3", "guid":"bro_3"},
   * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "method": "bro data 4", "ttl": "data 4", "guid":"bro_4"},
   * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "method": "bro data 5", "ttl": "data 5", "guid":"bro_5"}
   * ]
   */
  @Multiline
  public static String broData;

  /**
   * [
   * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "sig_generator": "sig_generator 1", "ttl": 1, "guid":"snort_1", "threat:triage:score":10.0},
   * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 2", "ttl": 2, "guid":"snort_2", "threat:triage:score":20.0},
   * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 3", "ttl": 3, "guid":"snort_3"},
   * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 4", "ttl": 4, "guid":"snort_4"},
   * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 5", "ttl": 5, "guid":"snort_5"}
   * ]
   */
  @Multiline
  public static String snortData;

  /**
   * {
   * "indices": ["bro", "snort", "some_collection"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String allQuery;

  /**
   * {
   * "guid": "bro_3",
   * "sensorType": "bro"
   * }
   */
  @Multiline
  public static String findOneGuidQuery;

  /**
   * [
   * {
   * "guid": "bro_1",
   * "sensorType": "bro"
   * },
   * {
   * "guid": "snort_2",
   * "sensorType": "snort"
   * }
   * ]
   */
  @Multiline
  public static String getAllLatestQuery;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "ip_src_addr:192.168.1.1",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String filterQuery;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "ip_src_port",
   *     "sortOrder": "asc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String sortQuery;

  /**
   * {
   *  "indices": [
   *    "snort",
   *    "bro"
   *  ],
   * "query": "*",
   * "from": 0,
   * "size": 25,
   * "sort": [
   *    {
   *      "field": "threat:triage:score",
   *      "sortOrder": "asc"
   *    }
   *  ]
   * }
   */
  @Multiline
  public static String sortAscendingWithMissingFields;

  /**
   * {
   *  "indices": [
   *    "snort",
   *    "bro"
   *  ],
   * "query": "*",
   * "from": 0,
   * "size": 25,
   * "sort": [
   *    {
   *      "field": "threat:triage:score",
   *      "sortOrder": "desc"
   *    }
   *  ]
   * }
   */
  @Multiline
  public static String sortDescendingWithMissingFields;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 4,
   * "size": 3,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String paginationQuery;

  /**
   * {
   * "indices": ["bro"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String indexQuery;

  /**
   * {
   * "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp", "latitude", "score", "is_alert"],
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String facetQueryRaw;

  /**
   * {
   * "facetFields": ["location_point"],
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String badFacetQuery;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String disabledFacetQuery;

  /**
   * {
   * "facetFields": ["sig_generator"],
   * "indices": ["bro", "snort"],
   * "query": "*:*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String missingTypeFacetQuery;

  /**
   * {
   * "facetFields": ["ttl"],
   * "indices": ["bro", "snort"],
   * "query": "*:*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String differentTypeFacetQuery;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 101,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String exceededMaxResultsQuery;

  /**
   * {
   * "fields": ["ip_src_addr"],
   * "indices": ["bro", "snort"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String fieldsQuery;

  /**
   * {
   * "fields": ["ip_src_addr"],
   * "indices": ["bro", "snort"],
   * "query": "ip_src_addr:192.168.1.9",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String noResultsFieldsQuery;

  /**
   * {
   * "fields": ["guid"],
   * "indices": ["bro"],
   * "query": "*",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "guid",
   *     "sortOrder": "asc"
   *   }
   * ]
   * }
   * }
   */
  @Multiline
  public static String sortByGuidQuery;

  /**
   * {
   * "groups": [
   *   {
   *     "field":"is_alert"
   *   },
   *   {
   *     "field":"latitude"
   *   }
   * ],
   * "scoreField":"score",
   * "indices": ["bro", "snort"],
   * "query": "*"
   * }
   */
  @Multiline
  public static String groupByQuery;

  /**
   * {
   * "groups": [
   *   {
   *     "field":"is_alert",
   *     "order": {
   *       "groupOrderType": "count",
   *       "sortOrder": "ASC"
   *     }
   *   },
   *   {
   *     "field":"ip_src_addr",
   *     "order": {
   *       "groupOrderType": "term",
   *       "sortOrder": "DESC"
   *     }
   *   }
   * ],
   * "indices": ["bro", "snort"],
   * "query": "*"
   * }
   */
  @Multiline
  public static String sortedGroupByQuery;

  /**
   * {
   * "groups": [
   *   {
   *     "field":"location_point"
   *   }
   * ],
   * "indices": ["bro", "snort"],
   * "query": "*"
   * }
   */
  @Multiline
  public static String badGroupQuery;

  /**
   * {
   * "groups": [
   *   {
   *     "field":"ip_src_addr",
   *     "order": {
   *       "groupOrderType": "term",
   *       "sortOrder": "DESC"
   *     }
   *   }
   * ],
   * "indices": ["bro", "snort"],
   * "query": "*"
   * }
   */
  @Multiline
  public static String groupByIpQuery;

  /**
   * {
   * "indices": ["bro", "snort"],
   * "query": "ttl:\"data 1\"",
   * "from": 0,
   * "size": 10,
   * "sort": [
   *   {
   *     "field": "timestamp",
   *     "sortOrder": "desc"
   *   }
   * ]
   * }
   */
  @Multiline
  public static String differentTypeFilterQuery;

  protected static InMemoryComponent indexComponent;

  @Rule
  public ExpectedException thrown = ExpectedException.none();

  @Test
  public void all_query_returns_all_results() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    Assert.assertEquals(10, results.size());
    for(int i = 0;i < 5;++i) {
      Assert.assertEquals("snort", results.get(i).getSource().get(getSourceTypeField()));
      Assert.assertEquals(getIndexName("snort"), results.get(i).getIndex());
      Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString());
    }
    for (int i = 5; i < 10; ++i) {
      Assert.assertEquals("bro", results.get(i).getSource().get(getSourceTypeField()));
      Assert.assertEquals(getIndexName("bro"), results.get(i).getIndex());
      Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString());
    }
  }

  @Test
  public void find_one_guid() throws Exception {
    GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class);
    Optional<Map<String, Object>> response = getIndexDao().getLatestResult(request);
    Assert.assertTrue(response.isPresent());
    Map<String, Object> doc = response.get();
    Assert.assertEquals("bro", doc.get(getSourceTypeField()));
    Assert.assertEquals("3", doc.get("timestamp").toString());
  }

  @Test
  public void get_all_latest_guid() throws Exception {
    List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new JSONUtils.ReferenceSupplier<List<GetRequest>>(){});
    Map<String, Document> docs = new HashMap<>();

    for(Document doc : getIndexDao().getAllLatest(request)) {
      docs.put(doc.getGuid(), doc);
    }
    Assert.assertEquals(2, docs.size());
    Assert.assertTrue(docs.keySet().contains("bro_1"));
    Assert.assertTrue(docs.keySet().contains("snort_2"));
    Assert.assertEquals("bro", docs.get("bro_1").getDocument().get(getSourceTypeField()));
    Assert.assertEquals("snort", docs.get("snort_2").getDocument().get(getSourceTypeField()));
  }

  @Test
  public void filter_query_filters_results() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(3, response.getTotal());
    List<SearchResult> results = response.getResults();
    Assert.assertEquals("snort", results.get(0).getSource().get(getSourceTypeField()));
    Assert.assertEquals("9", results.get(0).getSource().get("timestamp").toString());
    Assert.assertEquals("snort", results.get(1).getSource().get(getSourceTypeField()));
    Assert.assertEquals("7", results.get(1).getSource().get("timestamp").toString());
    Assert.assertEquals("bro", results.get(2).getSource().get(getSourceTypeField()));
    Assert.assertEquals("1", results.get(2).getSource().get("timestamp").toString());
  }

  @Test
  public void sort_query_sorts_results_ascending() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    for (int i = 8001; i < 8011; ++i) {
      Assert.assertEquals(i, results.get(i - 8001).getSource().get("ip_src_port"));
    }
  }

  @Test
  public void sort_ascending_with_missing_fields() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    Assert.assertEquals(10, results.size());

    // the remaining are missing the 'threat:triage:score' and should be sorted last
    for (int i = 0; i < 8; i++) {
      Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score"));
    }

    // validate sorted order - there are only 2 with a 'threat:triage:score'
    Assert.assertEquals("10.0", results.get(8).getSource().get("threat:triage:score").toString());
    Assert.assertEquals("20.0", results.get(9).getSource().get("threat:triage:score").toString());
  }

  @Test
  public void sort_descending_with_missing_fields() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    Assert.assertEquals(10, results.size());

    // validate sorted order - there are only 2 with a 'threat:triage:score'
    Assert.assertEquals("20.0", results.get(0).getSource().get("threat:triage:score").toString());
    Assert.assertEquals("10.0", results.get(1).getSource().get("threat:triage:score").toString());

    // the remaining are missing the 'threat:triage:score' and should be sorted last
    for (int i = 2; i < 10; i++) {
      Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score"));
    }
  }

  @Test
  public void results_are_paginated() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    Assert.assertEquals(3, results.size());
    Assert.assertEquals("snort", results.get(0).getSource().get(getSourceTypeField()));
    Assert.assertEquals("6", results.get(0).getSource().get("timestamp").toString());
    Assert.assertEquals("bro", results.get(1).getSource().get(getSourceTypeField()));
    Assert.assertEquals("5", results.get(1).getSource().get("timestamp").toString());
    Assert.assertEquals("bro", results.get(2).getSource().get(getSourceTypeField()));
    Assert.assertEquals("4", results.get(2).getSource().get("timestamp").toString());
  }

  @Test
  public void returns_results_only_for_specified_indices() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(5, response.getTotal());
    List<SearchResult> results = response.getResults();
    for (int i = 5, j = 0; i > 0; i--, j++) {
      Assert.assertEquals("bro", results.get(j).getSource().get(getSourceTypeField()));
      Assert.assertEquals(i + "", results.get(j).getSource().get("timestamp").toString());
    }
  }

  @Test
  public void facet_query_yields_field_types() throws Exception {
    String facetQuery = facetQueryRaw.replace("source:type", getSourceTypeField());
    SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(10, response.getTotal());
    Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
    Assert.assertEquals(8, facetCounts.size());
    Map<String, Long> sourceTypeCounts = facetCounts.get(getSourceTypeField());
    Assert.assertEquals(2, sourceTypeCounts.size());
    Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro"));
    Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort"));
    Map<String, Long> ipSrcAddrCounts = facetCounts.get("ip_src_addr");
    Assert.assertEquals(8, ipSrcAddrCounts.size());
    Assert.assertEquals(new Long(3), ipSrcAddrCounts.get("192.168.1.1"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.2"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.3"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.4"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.5"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.6"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.7"));
    Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.8"));
    Map<String, Long> ipSrcPortCounts = facetCounts.get("ip_src_port");
    Assert.assertEquals(10, ipSrcPortCounts.size());
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8001"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8002"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8003"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8004"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8005"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8006"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8007"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8008"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8009"));
    Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8010"));
    Map<String, Long> longFieldCounts = facetCounts.get("long_field");
    Assert.assertEquals(2, longFieldCounts.size());
    Assert.assertEquals(new Long(8), longFieldCounts.get("10000"));
    Assert.assertEquals(new Long(2), longFieldCounts.get("20000"));
    Map<String, Long> timestampCounts = facetCounts.get("timestamp");
    Assert.assertEquals(10, timestampCounts.size());
    Assert.assertEquals(new Long(1), timestampCounts.get("1"));
    Assert.assertEquals(new Long(1), timestampCounts.get("2"));
    Assert.assertEquals(new Long(1), timestampCounts.get("3"));
    Assert.assertEquals(new Long(1), timestampCounts.get("4"));
    Assert.assertEquals(new Long(1), timestampCounts.get("5"));
    Assert.assertEquals(new Long(1), timestampCounts.get("6"));
    Assert.assertEquals(new Long(1), timestampCounts.get("7"));
    Assert.assertEquals(new Long(1), timestampCounts.get("8"));
    Assert.assertEquals(new Long(1), timestampCounts.get("9"));
    Assert.assertEquals(new Long(1), timestampCounts.get("10"));
    Map<String, Long> latitudeCounts = facetCounts.get("latitude");
    Assert.assertEquals(2, latitudeCounts.size());
    List<String> latitudeKeys = new ArrayList<>(latitudeCounts.keySet());
    Collections.sort(latitudeKeys);
    Assert.assertEquals(48.0001, Double.parseDouble(latitudeKeys.get(0)), 0.00001);
    Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001);
    Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0)));
    Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1)));
    Map<String, Long> scoreFieldCounts = facetCounts.get("score");
    Assert.assertEquals(4, scoreFieldCounts.size());
    List<String> scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet());
    Collections.sort(scoreFieldKeys);
    Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001);
    Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001);
    Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001);
    Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001);
    Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0)));
    Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1)));
    Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2)));
    Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3)));
    Map<String, Long> isAlertCounts = facetCounts.get("is_alert");
    Assert.assertEquals(2, isAlertCounts.size());
    Assert.assertEquals(new Long(6), isAlertCounts.get("true"));
    Assert.assertEquals(new Long(4), isAlertCounts.get("false"));
  }

  @Test
  public void disabled_facet_query_returns_null_count() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertNull(response.getFacetCounts());
  }

  @Test
  public void missing_type_facet_query() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(missingTypeFacetQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(10, response.getTotal());

    Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
    Assert.assertEquals(1, facetCounts.size());
    Map<String, Long> snortFieldCounts = facetCounts.get("sig_generator");
    Assert.assertEquals(5, snortFieldCounts.size());

    Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 5").longValue());
    Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 4").longValue());
    Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 3").longValue());
    Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 2").longValue());
    Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 1").longValue());
    response.getFacetCounts();
  }

  @Test
  public void different_type_facet_query() throws Exception {
    thrown.expect(Exception.class);
    SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(3, response.getTotal());
  }

  @Test
  public void exceeding_max_results_throws_exception() throws Exception {
    thrown.expect(InvalidSearchException.class);
    thrown.expectMessage("Search result size must be less than 100");
    SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class);
    getIndexDao().search(request);
  }

  @Test
  public void column_metadata_for_missing_index() throws Exception {
    // getColumnMetadata with an index that doesn't exist
    {
      Map<String, FieldType> fieldTypes = getIndexDao().getColumnMetadata(Collections.singletonList("someindex"));
      Assert.assertEquals(0, fieldTypes.size());
    }
  }

  @Test
  public void no_results_returned_when_query_does_not_match() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(0, response.getTotal());
  }

  @Test
  public void group_by_ip_query() throws Exception {
    GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class);
    GroupResponse response = getIndexDao().group(request);

    // expect only 1 group for 'ip_src_addr'
    Assert.assertEquals("ip_src_addr", response.getGroupedBy());

    // there are 8 different 'ip_src_addr' values
    List<GroupResult> groups = response.getGroupResults();
    Assert.assertEquals(8, groups.size());

    // expect dotted-decimal notation in descending order
    Assert.assertEquals("192.168.1.8", groups.get(0).getKey());
    Assert.assertEquals("192.168.1.7", groups.get(1).getKey());
    Assert.assertEquals("192.168.1.6", groups.get(2).getKey());
    Assert.assertEquals("192.168.1.5", groups.get(3).getKey());
    Assert.assertEquals("192.168.1.4", groups.get(4).getKey());
    Assert.assertEquals("192.168.1.3", groups.get(5).getKey());
    Assert.assertEquals("192.168.1.2", groups.get(6).getKey());
    Assert.assertEquals("192.168.1.1", groups.get(7).getKey());
  }

  @Test
  public void group_by_returns_results_in_groups() throws Exception {
    // Group by test case, default order is count descending
    GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class);
    GroupResponse response = getIndexDao().group(request);
    Assert.assertEquals("is_alert", response.getGroupedBy());
    List<GroupResult> isAlertGroups = response.getGroupResults();
    Assert.assertEquals(2, isAlertGroups.size());

    // isAlert == true group
    GroupResult trueGroup = isAlertGroups.get(0);
    Assert.assertEquals("true", trueGroup.getKey());
    Assert.assertEquals(6, trueGroup.getTotal());
    Assert.assertEquals("latitude", trueGroup.getGroupedBy());
    Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001);
    List<GroupResult> trueLatitudeGroups = trueGroup.getGroupResults();
    Assert.assertEquals(2, trueLatitudeGroups.size());


    // isAlert == true && latitude == 48.5839 group
    GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0);
    Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001);
    Assert.assertEquals(5, trueLatitudeGroup2.getTotal());
    Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001);

    // isAlert == true && latitude == 48.0001 group
    GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1);
    Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001);
    Assert.assertEquals(1, trueLatitudeGroup1.getTotal());
    Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001);

    // isAlert == false group
    GroupResult falseGroup = isAlertGroups.get(1);
    Assert.assertEquals("false", falseGroup.getKey());
    Assert.assertEquals("latitude", falseGroup.getGroupedBy());
    Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001);
    List<GroupResult> falseLatitudeGroups = falseGroup.getGroupResults();
    Assert.assertEquals(2, falseLatitudeGroups.size());

    // isAlert == false && latitude == 48.5839 group
    GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0);
    Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001);
    Assert.assertEquals(3, falseLatitudeGroup2.getTotal());
    Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001);

    // isAlert == false && latitude == 48.0001 group
    GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1);
    Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001);
    Assert.assertEquals(1, falseLatitudeGroup1.getTotal());
    Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001);
  }

  @Test
  public void group_by_returns_results_in_sorted_groups() throws Exception {
    // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending
    GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class);
    GroupResponse response = getIndexDao().group(request);
    Assert.assertEquals("is_alert", response.getGroupedBy());
    List<GroupResult> isAlertGroups = response.getGroupResults();
    Assert.assertEquals(2, isAlertGroups.size());

    // isAlert == false group
    GroupResult falseGroup = isAlertGroups.get(0);
    Assert.assertEquals(4, falseGroup.getTotal());
    Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy());
    List<GroupResult> falseIpSrcAddrGroups = falseGroup.getGroupResults();
    Assert.assertEquals(4, falseIpSrcAddrGroups.size());

    // isAlert == false && ip_src_addr == 192.168.1.8 group
    GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0);
    Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey());
    Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal());
    Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy());
    Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.7 group
    GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1);
    Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey());
    Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal());
    Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy());
    Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.6 group
    GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2);
    Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey());
    Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal());
    Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy());
    Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.2 group
    GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3);
    Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey());
    Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal());
    Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy());
    Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults());

    // isAlert == false group
    GroupResult trueGroup = isAlertGroups.get(1);
    Assert.assertEquals(6, trueGroup.getTotal());
    Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy());
    List<GroupResult> trueIpSrcAddrGroups = trueGroup.getGroupResults();
    Assert.assertEquals(4, trueIpSrcAddrGroups.size());

    // isAlert == false && ip_src_addr == 192.168.1.5 group
    GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0);
    Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey());
    Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal());
    Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy());
    Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.4 group
    GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1);
    Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey());
    Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal());
    Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy());
    Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.3 group
    GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2);
    Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey());
    Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal());
    Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy());
    Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults());

    // isAlert == false && ip_src_addr == 192.168.1.1 group
    GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3);
    Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey());
    Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal());
    Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy());
    Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults());
  }

  @Test
  public void queries_fields() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(10, response.getTotal());
    List<SearchResult> results = response.getResults();
    for (int i = 0; i < 5; ++i) {
      Map<String, Object> source = results.get(i).getSource();
      Assert.assertEquals(1, source.size());
      Assert.assertNotNull(source.get("ip_src_addr"));
    }
    for (int i = 5; i < 10; ++i) {
      Map<String, Object> source = results.get(i).getSource();
      Assert.assertEquals(1, source.size());
      Assert.assertNotNull(source.get("ip_src_addr"));
    }
  }

  @Test
  public void sort_by_guid() throws Exception {
    SearchRequest request = JSONUtils.INSTANCE.load(sortByGuidQuery, SearchRequest.class);
    SearchResponse response = getIndexDao().search(request);
    Assert.assertEquals(5, response.getTotal());
    List<SearchResult> results = response.getResults();
    for (int i = 0; i < 5; ++i) {
      Map<String, Object> source = results.get(i).getSource();
      Assert.assertEquals(1, source.size());
      Assert.assertEquals(source.get("guid"), "bro_" + (i + 1));
    }
  }

  @AfterClass
  public static void stop() {
    indexComponent.stop();
  }

  @Test
  public abstract void returns_column_data_for_multiple_indices() throws Exception;
  @Test
  public abstract void returns_column_metadata_for_specified_indices() throws Exception;
  @Test
  public abstract void different_type_filter_query() throws Exception;

  protected abstract IndexDao getIndexDao();

  protected abstract String getSourceTypeField();

  protected abstract String getIndexName(String sensorType);
}