| """Unit tests for Superset""" |
| from __future__ import absolute_import |
| from __future__ import division |
| from __future__ import print_function |
| from __future__ import unicode_literals |
| |
| from datetime import datetime |
| import json |
| import unittest |
| |
| from mock import Mock, patch |
| |
| from superset import db, security, sm |
| from superset.connectors.druid.models import ( |
| DruidCluster, DruidDatasource, DruidMetric, |
| ) |
| from .base_tests import SupersetTestCase |
| |
| |
| class PickableMock(Mock): |
| def __reduce__(self): |
| return (Mock, ()) |
| |
| |
| SEGMENT_METADATA = [{ |
| 'id': 'some_id', |
| 'intervals': ['2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z'], |
| 'columns': { |
| '__time': { |
| 'type': 'LONG', 'hasMultipleValues': False, |
| 'size': 407240380, 'cardinality': None, 'errorMessage': None}, |
| 'dim1': { |
| 'type': 'STRING', 'hasMultipleValues': False, |
| 'size': 100000, 'cardinality': 1944, 'errorMessage': None}, |
| 'dim2': { |
| 'type': 'STRING', 'hasMultipleValues': True, |
| 'size': 100000, 'cardinality': 1504, 'errorMessage': None}, |
| 'metric1': { |
| 'type': 'FLOAT', 'hasMultipleValues': False, |
| 'size': 100000, 'cardinality': None, 'errorMessage': None}, |
| }, |
| 'aggregators': { |
| 'metric1': { |
| 'type': 'longSum', |
| 'name': 'metric1', |
| 'fieldName': 'metric1'}, |
| }, |
| 'size': 300000, |
| 'numRows': 5000000, |
| }] |
| |
| GB_RESULT_SET = [ |
| { |
| 'version': 'v1', |
| 'timestamp': '2012-01-01T00:00:00.000Z', |
| 'event': { |
| 'dim1': 'Canada', |
| 'metric1': 12345678, |
| }, |
| }, |
| { |
| 'version': 'v1', |
| 'timestamp': '2012-01-01T00:00:00.000Z', |
| 'event': { |
| 'dim1': 'USA', |
| 'metric1': 12345678 / 2, |
| }, |
| }, |
| ] |
| |
| |
| class DruidTests(SupersetTestCase): |
| |
| """Testing interactions with Druid""" |
| |
| def __init__(self, *args, **kwargs): |
| super(DruidTests, self).__init__(*args, **kwargs) |
| |
| @patch('superset.connectors.druid.models.PyDruid') |
| def test_client(self, PyDruid): |
| self.login(username='admin') |
| instance = PyDruid.return_value |
| instance.time_boundary.return_value = [ |
| {'result': {'maxTime': '2016-01-01'}}] |
| instance.segment_metadata.return_value = SEGMENT_METADATA |
| |
| cluster = ( |
| db.session |
| .query(DruidCluster) |
| .filter_by(cluster_name='test_cluster') |
| .first() |
| ) |
| if cluster: |
| db.session.delete(cluster) |
| db.session.commit() |
| |
| cluster = DruidCluster( |
| cluster_name='test_cluster', |
| coordinator_host='localhost', |
| coordinator_port=7979, |
| broker_host='localhost', |
| broker_port=7980, |
| metadata_last_refreshed=datetime.now()) |
| |
| db.session.add(cluster) |
| cluster.get_datasources = PickableMock(return_value=['test_datasource']) |
| cluster.get_druid_version = PickableMock(return_value='0.9.1') |
| cluster.refresh_datasources() |
| cluster.refresh_datasources(merge_flag=True) |
| datasource_id = cluster.datasources[0].id |
| db.session.commit() |
| |
| nres = [ |
| list(v['event'].items()) + [('timestamp', v['timestamp'])] |
| for v in GB_RESULT_SET] |
| nres = [dict(v) for v in nres] |
| import pandas as pd |
| df = pd.DataFrame(nres) |
| instance.export_pandas.return_value = df |
| instance.query_dict = {} |
| instance.query_builder.last_query.query_dict = {} |
| |
| resp = self.get_resp('/superset/explore/druid/{}/'.format( |
| datasource_id)) |
| self.assertIn('test_datasource', resp) |
| form_data = { |
| 'viz_type': 'table', |
| 'granularity': 'one+day', |
| 'druid_time_origin': '', |
| 'since': '7+days+ago', |
| 'until': 'now', |
| 'row_limit': 5000, |
| 'include_search': 'false', |
| 'metrics': ['count'], |
| 'groupby': ['dim1'], |
| 'force': 'true', |
| } |
| # One groupby |
| url = ( |
| '/superset/explore_json/druid/{}/?form_data={}'.format( |
| datasource_id, json.dumps(form_data)) |
| ) |
| resp = self.get_json_resp(url) |
| self.assertEqual('Canada', resp['data']['records'][0]['dim1']) |
| |
| form_data = { |
| 'viz_type': 'table', |
| 'granularity': 'one+day', |
| 'druid_time_origin': '', |
| 'since': '7+days+ago', |
| 'until': 'now', |
| 'row_limit': 5000, |
| 'include_search': 'false', |
| 'metrics': ['count'], |
| 'groupby': ['dim1', 'dim2d'], |
| 'force': 'true', |
| } |
| # two groupby |
| url = ( |
| '/superset/explore_json/druid/{}/?form_data={}'.format( |
| datasource_id, json.dumps(form_data)) |
| ) |
| resp = self.get_json_resp(url) |
| self.assertEqual('Canada', resp['data']['records'][0]['dim1']) |
| |
| def test_druid_sync_from_config(self): |
| CLUSTER_NAME = 'new_druid' |
| self.login() |
| cluster = self.get_or_create( |
| DruidCluster, |
| {'cluster_name': CLUSTER_NAME}, |
| db.session) |
| |
| db.session.merge(cluster) |
| db.session.commit() |
| |
| ds = ( |
| db.session.query(DruidDatasource) |
| .filter_by(datasource_name='test_click') |
| .first() |
| ) |
| if ds: |
| db.session.delete(ds) |
| db.session.commit() |
| |
| cfg = { |
| 'user': 'admin', |
| 'cluster': CLUSTER_NAME, |
| 'config': { |
| 'name': 'test_click', |
| 'dimensions': ['affiliate_id', 'campaign', 'first_seen'], |
| 'metrics_spec': [{'type': 'count', 'name': 'count'}, |
| {'type': 'sum', 'name': 'sum'}], |
| 'batch_ingestion': { |
| 'sql': "SELECT * FROM clicks WHERE d='{{ ds }}'", |
| 'ts_column': 'd', |
| 'sources': [{ |
| 'table': 'clicks', |
| 'partition': "d='{{ ds }}'", |
| }], |
| }, |
| }, |
| } |
| |
| def check(): |
| resp = self.client.post('/superset/sync_druid/', data=json.dumps(cfg)) |
| druid_ds = ( |
| db.session |
| .query(DruidDatasource) |
| .filter_by(datasource_name='test_click') |
| .one() |
| ) |
| col_names = set([c.column_name for c in druid_ds.columns]) |
| assert {'affiliate_id', 'campaign', 'first_seen'} == col_names |
| metric_names = {m.metric_name for m in druid_ds.metrics} |
| assert {'count', 'sum'} == metric_names |
| assert resp.status_code == 201 |
| |
| check() |
| # checking twice to make sure a second sync yields the same results |
| check() |
| |
| # datasource exists, add new metrics and dimensions |
| cfg = { |
| 'user': 'admin', |
| 'cluster': CLUSTER_NAME, |
| 'config': { |
| 'name': 'test_click', |
| 'dimensions': ['affiliate_id', 'second_seen'], |
| 'metrics_spec': [ |
| {'type': 'bla', 'name': 'sum'}, |
| {'type': 'unique', 'name': 'unique'}, |
| ], |
| }, |
| } |
| resp = self.client.post('/superset/sync_druid/', data=json.dumps(cfg)) |
| druid_ds = db.session.query(DruidDatasource).filter_by( |
| datasource_name='test_click').one() |
| # columns and metrics are not deleted if config is changed as |
| # user could define his own dimensions / metrics and want to keep them |
| assert set([c.column_name for c in druid_ds.columns]) == set( |
| ['affiliate_id', 'campaign', 'first_seen', 'second_seen']) |
| assert set([m.metric_name for m in druid_ds.metrics]) == set( |
| ['count', 'sum', 'unique']) |
| # metric type will not be overridden, sum stays instead of bla |
| assert set([m.metric_type for m in druid_ds.metrics]) == set( |
| ['longSum', 'sum', 'unique']) |
| assert resp.status_code == 201 |
| |
| def test_filter_druid_datasource(self): |
| CLUSTER_NAME = 'new_druid' |
| cluster = self.get_or_create( |
| DruidCluster, |
| {'cluster_name': CLUSTER_NAME}, |
| db.session) |
| db.session.merge(cluster) |
| |
| gamma_ds = self.get_or_create( |
| DruidDatasource, {'datasource_name': 'datasource_for_gamma'}, |
| db.session) |
| gamma_ds.cluster = cluster |
| db.session.merge(gamma_ds) |
| |
| no_gamma_ds = self.get_or_create( |
| DruidDatasource, {'datasource_name': 'datasource_not_for_gamma'}, |
| db.session) |
| no_gamma_ds.cluster = cluster |
| db.session.merge(no_gamma_ds) |
| db.session.commit() |
| |
| security.merge_perm(sm, 'datasource_access', gamma_ds.perm) |
| security.merge_perm(sm, 'datasource_access', no_gamma_ds.perm) |
| |
| perm = sm.find_permission_view_menu( |
| 'datasource_access', gamma_ds.get_perm()) |
| sm.add_permission_role(sm.find_role('Gamma'), perm) |
| sm.get_session.commit() |
| |
| self.login(username='gamma') |
| url = '/druiddatasourcemodelview/list/' |
| resp = self.get_resp(url) |
| self.assertIn('datasource_for_gamma', resp) |
| self.assertNotIn('datasource_not_for_gamma', resp) |
| |
| @patch('superset.connectors.druid.models.PyDruid') |
| def test_sync_druid_perm(self, PyDruid): |
| self.login(username='admin') |
| instance = PyDruid.return_value |
| instance.time_boundary.return_value = [ |
| {'result': {'maxTime': '2016-01-01'}}] |
| instance.segment_metadata.return_value = SEGMENT_METADATA |
| |
| cluster = ( |
| db.session |
| .query(DruidCluster) |
| .filter_by(cluster_name='test_cluster') |
| .first() |
| ) |
| if cluster: |
| db.session.delete(cluster) |
| db.session.commit() |
| |
| cluster = DruidCluster( |
| cluster_name='test_cluster', |
| coordinator_host='localhost', |
| coordinator_port=7979, |
| broker_host='localhost', |
| broker_port=7980, |
| metadata_last_refreshed=datetime.now()) |
| |
| db.session.add(cluster) |
| cluster.get_datasources = PickableMock( |
| return_value=['test_datasource'], |
| ) |
| cluster.get_druid_version = PickableMock(return_value='0.9.1') |
| |
| cluster.refresh_datasources() |
| cluster.datasources[0].merge_flag = True |
| metadata = cluster.datasources[0].latest_metadata() |
| self.assertEqual(len(metadata), 4) |
| db.session.commit() |
| |
| view_menu_name = cluster.datasources[0].get_perm() |
| view_menu = sm.find_view_menu(view_menu_name) |
| permission = sm.find_permission('datasource_access') |
| |
| pv = sm.get_session.query(sm.permissionview_model).filter_by( |
| permission=permission, view_menu=view_menu).first() |
| assert pv is not None |
| |
| def test_metrics_and_post_aggs(self): |
| """ |
| Test generation of metrics and post-aggregations from an initial list |
| of superset metrics (which may include the results of either). This |
| primarily tests that specifying a post-aggregator metric will also |
| require the raw aggregation of the associated druid metric column. |
| """ |
| metrics_dict = { |
| 'unused_count': DruidMetric( |
| metric_name='unused_count', |
| verbose_name='COUNT(*)', |
| metric_type='count', |
| json=json.dumps({'type': 'count', 'name': 'unused_count'}), |
| ), |
| 'some_sum': DruidMetric( |
| metric_name='some_sum', |
| verbose_name='SUM(*)', |
| metric_type='sum', |
| json=json.dumps({'type': 'sum', 'name': 'sum'}), |
| ), |
| 'a_histogram': DruidMetric( |
| metric_name='a_histogram', |
| verbose_name='APPROXIMATE_HISTOGRAM(*)', |
| metric_type='approxHistogramFold', |
| json=json.dumps( |
| {'type': 'approxHistogramFold', 'name': 'a_histogram'}, |
| ), |
| ), |
| 'aCustomMetric': DruidMetric( |
| metric_name='aCustomMetric', |
| verbose_name='MY_AWESOME_METRIC(*)', |
| metric_type='aCustomType', |
| json=json.dumps( |
| {'type': 'customMetric', 'name': 'aCustomMetric'}, |
| ), |
| ), |
| 'quantile_p95': DruidMetric( |
| metric_name='quantile_p95', |
| verbose_name='P95(*)', |
| metric_type='postagg', |
| json=json.dumps({ |
| 'type': 'quantile', |
| 'probability': 0.95, |
| 'name': 'p95', |
| 'fieldName': 'a_histogram', |
| }), |
| ), |
| 'aCustomPostAgg': DruidMetric( |
| metric_name='aCustomPostAgg', |
| verbose_name='CUSTOM_POST_AGG(*)', |
| metric_type='postagg', |
| json=json.dumps({ |
| 'type': 'customPostAgg', |
| 'name': 'aCustomPostAgg', |
| 'field': { |
| 'type': 'fieldAccess', |
| 'fieldName': 'aCustomMetric', |
| }, |
| }), |
| ), |
| } |
| |
| metrics = ['some_sum'] |
| all_metrics, post_aggs = DruidDatasource._metrics_and_post_aggs( |
| metrics, metrics_dict) |
| |
| assert all_metrics == ['some_sum'] |
| assert post_aggs == {} |
| |
| metrics = ['quantile_p95'] |
| all_metrics, post_aggs = DruidDatasource._metrics_and_post_aggs( |
| metrics, metrics_dict) |
| |
| result_postaggs = set(['quantile_p95']) |
| assert all_metrics == ['a_histogram'] |
| assert set(post_aggs.keys()) == result_postaggs |
| |
| metrics = ['aCustomPostAgg'] |
| all_metrics, post_aggs = DruidDatasource._metrics_and_post_aggs( |
| metrics, metrics_dict) |
| |
| result_postaggs = set(['aCustomPostAgg']) |
| assert all_metrics == ['aCustomMetric'] |
| assert set(post_aggs.keys()) == result_postaggs |
| |
| def test_get_filters_ignores_invalid_filter_objects(self): |
| filtr = {'col': 'col1', 'op': '=='} |
| filters = [filtr] |
| self.assertEqual(None, DruidDatasource.get_filters(filters, [])) |
| |
| def test_get_filters_constructs_filter_in(self): |
| filtr = {'col': 'A', 'op': 'in', 'val': ['a', 'b', 'c']} |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertIn('filter', res.filter) |
| self.assertIn('fields', res.filter['filter']) |
| self.assertEqual('or', res.filter['filter']['type']) |
| self.assertEqual(3, len(res.filter['filter']['fields'])) |
| |
| def test_get_filters_constructs_filter_not_in(self): |
| filtr = {'col': 'A', 'op': 'not in', 'val': ['a', 'b', 'c']} |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertIn('filter', res.filter) |
| self.assertIn('type', res.filter['filter']) |
| self.assertEqual('not', res.filter['filter']['type']) |
| self.assertIn('field', res.filter['filter']) |
| self.assertEqual( |
| 3, |
| len(res.filter['filter']['field'].filter['filter']['fields']), |
| ) |
| |
| def test_get_filters_constructs_filter_equals(self): |
| filtr = {'col': 'A', 'op': '==', 'val': 'h'} |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertEqual('selector', res.filter['filter']['type']) |
| self.assertEqual('A', res.filter['filter']['dimension']) |
| self.assertEqual('h', res.filter['filter']['value']) |
| |
| def test_get_filters_constructs_filter_not_equals(self): |
| filtr = {'col': 'A', 'op': '!=', 'val': 'h'} |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertEqual('not', res.filter['filter']['type']) |
| self.assertEqual( |
| 'h', |
| res.filter['filter']['field'].filter['filter']['value'], |
| ) |
| |
| def test_get_filters_constructs_bounds_filter(self): |
| filtr = {'col': 'A', 'op': '>=', 'val': 'h'} |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertFalse(res.filter['filter']['lowerStrict']) |
| self.assertEqual('A', res.filter['filter']['dimension']) |
| self.assertEqual('h', res.filter['filter']['lower']) |
| self.assertFalse(res.filter['filter']['alphaNumeric']) |
| filtr['op'] = '>' |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertTrue(res.filter['filter']['lowerStrict']) |
| filtr['op'] = '<=' |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertFalse(res.filter['filter']['upperStrict']) |
| self.assertEqual('h', res.filter['filter']['upper']) |
| filtr['op'] = '<' |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertTrue(res.filter['filter']['upperStrict']) |
| |
| def test_get_filters_constructs_regex_filter(self): |
| filtr = {'col': 'A', 'op': 'regex', 'val': '[abc]'} |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertEqual('regex', res.filter['filter']['type']) |
| self.assertEqual('[abc]', res.filter['filter']['pattern']) |
| self.assertEqual('A', res.filter['filter']['dimension']) |
| |
| def test_get_filters_composes_multiple_filters(self): |
| filtr1 = {'col': 'A', 'op': '!=', 'val': 'y'} |
| filtr2 = {'col': 'B', 'op': 'in', 'val': ['a', 'b', 'c']} |
| res = DruidDatasource.get_filters([filtr1, filtr2], []) |
| self.assertEqual('and', res.filter['filter']['type']) |
| self.assertEqual(2, len(res.filter['filter']['fields'])) |
| |
| def test_get_filters_ignores_in_not_in_with_empty_value(self): |
| filtr1 = {'col': 'A', 'op': 'in', 'val': []} |
| filtr2 = {'col': 'A', 'op': 'not in', 'val': []} |
| res = DruidDatasource.get_filters([filtr1, filtr2], []) |
| self.assertEqual(None, res) |
| |
| def test_get_filters_constructs_equals_for_in_not_in_single_value(self): |
| filtr = {'col': 'A', 'op': 'in', 'val': ['a']} |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertEqual('selector', res.filter['filter']['type']) |
| |
| def test_get_filters_handles_arrays_for_string_types(self): |
| filtr = {'col': 'A', 'op': '==', 'val': ['a', 'b']} |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertEqual('a', res.filter['filter']['value']) |
| filtr = {'col': 'A', 'op': '==', 'val': []} |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertEqual('', res.filter['filter']['value']) |
| |
| def test_get_filters_handles_none_for_string_types(self): |
| filtr = {'col': 'A', 'op': '==', 'val': None} |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertEqual('', res.filter['filter']['value']) |
| |
| def test_get_filters_extracts_values_in_quotes(self): |
| filtr = {'col': 'A', 'op': 'in', 'val': [' "a" ']} |
| res = DruidDatasource.get_filters([filtr], []) |
| self.assertEqual('a', res.filter['filter']['value']) |
| |
| def test_get_filters_converts_strings_to_num(self): |
| filtr = {'col': 'A', 'op': 'in', 'val': ['6']} |
| res = DruidDatasource.get_filters([filtr], ['A']) |
| self.assertEqual(6, res.filter['filter']['value']) |
| filtr = {'col': 'A', 'op': '==', 'val': '6'} |
| res = DruidDatasource.get_filters([filtr], ['A']) |
| self.assertEqual(6, res.filter['filter']['value']) |
| |
| |
| if __name__ == '__main__': |
| unittest.main() |
