| ''' |
| Licensed to the Apache Software Foundation (ASF) under one |
| or more contributor license agreements. See the NOTICE file |
| distributed with this work for additional information |
| regarding copyright ownership. The ASF licenses this file |
| to you under the Apache License, Version 2.0 (the |
| "License"); you may not use this file except in compliance |
| with the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| ''' |
| |
| import json |
| import os |
| from unittest import TestCase |
| from mock.mock import patch |
| import socket |
| |
| class TestHDP26StackAdvisor(TestCase): |
| def setUp(self): |
| import imp |
| self.maxDiff = None |
| self.testDirectory = os.path.dirname(os.path.abspath(__file__)) |
| stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py') |
| hdp206StackAdvisorPath = os.path.join(self.testDirectory, |
| '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py') |
| hdp21StackAdvisorPath = os.path.join(self.testDirectory, |
| '../../../../../main/resources/stacks/HDP/2.1/services/stack_advisor.py') |
| hdp22StackAdvisorPath = os.path.join(self.testDirectory, |
| '../../../../../main/resources/stacks/HDP/2.2/services/stack_advisor.py') |
| hdp23StackAdvisorPath = os.path.join(self.testDirectory, |
| '../../../../../main/resources/stacks/HDP/2.3/services/stack_advisor.py') |
| hdp24StackAdvisorPath = os.path.join(self.testDirectory, |
| '../../../../../main/resources/stacks/HDP/2.4/services/stack_advisor.py') |
| hdp25StackAdvisorPath = os.path.join(self.testDirectory, |
| '../../../../../main/resources/stacks/HDP/2.5/services/stack_advisor.py') |
| hdp26StackAdvisorPath = os.path.join(self.testDirectory, |
| '../../../../../main/resources/stacks/HDP/2.6/services/stack_advisor.py') |
| hdp26StackAdvisorClassName = 'HDP26StackAdvisor' |
| |
| with open(stackAdvisorPath, 'rb') as fp: |
| imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) |
| with open(hdp206StackAdvisorPath, 'rb') as fp: |
| imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) |
| with open(hdp21StackAdvisorPath, 'rb') as fp: |
| imp.load_module('stack_advisor_impl', fp, hdp21StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) |
| with open(hdp22StackAdvisorPath, 'rb') as fp: |
| imp.load_module('stack_advisor_impl', fp, hdp22StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) |
| with open(hdp23StackAdvisorPath, 'rb') as fp: |
| imp.load_module('stack_advisor_impl', fp, hdp23StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) |
| with open(hdp24StackAdvisorPath, 'rb') as fp: |
| imp.load_module('stack_advisor_impl', fp, hdp24StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) |
| with open(hdp25StackAdvisorPath, 'rb') as fp: |
| imp.load_module('stack_advisor_impl', fp, hdp25StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE)) |
| with open(hdp26StackAdvisorPath, 'rb') as fp: |
| stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp26StackAdvisorPath, |
| ('.py', 'rb', imp.PY_SOURCE)) |
| clazz = getattr(stack_advisor_impl, hdp26StackAdvisorClassName) |
| self.stackAdvisor = clazz() |
| |
| # substitute method in the instance |
| self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid |
| self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic |
| |
| @patch('__builtin__.open') |
| @patch('os.path.exists') |
| def get_system_min_uid_magic(self, exists_mock, open_mock): |
| class MagicFile(object): |
| def read(self): |
| return """ |
| #test line UID_MIN 200 |
| UID_MIN 500 |
| """ |
| |
| def __exit__(self, exc_type, exc_val, exc_tb): |
| pass |
| |
| def __enter__(self): |
| return self |
| |
| exists_mock.return_value = True |
| open_mock.return_value = MagicFile() |
| return self.get_system_min_uid_real() |
| |
| def test_recommendDruidConfigurations_withMysql(self): |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 4, |
| "total_mem": 50331648, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| }, |
| } |
| ] |
| } |
| |
| services = { |
| "Versions": { |
| "parent_stack_version": "2.5", |
| "stack_name": "HDP", |
| "stack_version": "2.6", |
| "stack_hierarchy": { |
| "stack_name": "HDP", |
| "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"] |
| } |
| }, |
| "services": [{ |
| "StackServices": { |
| "service_name": "DRUID", |
| }, |
| "components": [ |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_COORDINATOR", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_OVERLORD", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_BROKER", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_HISTORICAL", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_MIDDLEMANAGER", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| } |
| ] |
| } |
| ], |
| "configurations": { |
| "druid-common": { |
| "properties": { |
| "database_name": "druid", |
| "metastore_hostname": "c6401.ambari.apache.org", |
| "druid.metadata.storage.type": "mysql", |
| "druid.extensions.loadList": "[\"postgresql-metadata-storage\"]" |
| } |
| } |
| } |
| } |
| |
| clusterData = { |
| "cpu": 4, |
| "mapMemory": 30000, |
| "amMemory": 20000, |
| "reduceMemory": 20560, |
| "containers": 30, |
| "ramPerContainer": 512, |
| "referenceNodeManagerHost": { |
| "total_mem": 10240 * 1024 |
| } |
| } |
| |
| configurations = { |
| } |
| |
| self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, |
| {'druid-historical': { |
| 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, |
| 'druid-broker': { |
| 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, |
| 'druid-common': {'properties': {'druid.extensions.loadList': '["mysql-metadata-storage"]', |
| 'druid.metadata.storage.connector.port': '3306', |
| 'druid.metadata.storage.connector.connectURI': 'jdbc:mysql://c6401.ambari.apache.org:3306/druid?createDatabaseIfNotExist=true', |
| 'druid.zk.service.host': ''}}, |
| 'druid-env': {'properties': {}, |
| 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'}, |
| 'druid.overlord.jvm.heap.memory': {'maximum': '49152'}, |
| 'druid.middlemanager.jvm.heap.memory': { |
| 'maximum': '49152'}, |
| 'druid.historical.jvm.heap.memory': {'maximum': '49152'}, |
| 'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}} |
| ) |
| |
| def test_recommendDruidConfigurations_WithPostgresql(self): |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 4, |
| "total_mem": 50331648, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| } |
| } |
| ] |
| } |
| |
| services = { |
| "Versions": { |
| "parent_stack_version": "2.5", |
| "stack_name": "HDP", |
| "stack_version": "2.6", |
| "stack_hierarchy": { |
| "stack_name": "HDP", |
| "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"] |
| } |
| }, |
| "services": [{ |
| "StackServices": { |
| "service_name": "DRUID", |
| }, |
| "components": [ |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_COORDINATOR", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_OVERLORD", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_BROKER", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_HISTORICAL", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_MIDDLEMANAGER", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| } |
| ] |
| } |
| ], |
| "configurations": { |
| "druid-common": { |
| "properties": { |
| "database_name": "druid", |
| "metastore_hostname": "c6401.ambari.apache.org", |
| "druid.metadata.storage.type": "postgresql", |
| "druid.extensions.loadList": "[\"mysql-metadata-storage\"]", |
| "druid.extensions.pullList": "[]" |
| } |
| } |
| } |
| } |
| |
| clusterData = { |
| } |
| |
| configurations = { |
| } |
| |
| self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, |
| {'druid-historical': { |
| 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, |
| 'druid-broker': { |
| 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, |
| 'druid-common': {'properties': {'druid.extensions.loadList': '["postgresql-metadata-storage"]', |
| 'druid.metadata.storage.connector.port': '5432', |
| 'druid.metadata.storage.connector.connectURI': 'jdbc:postgresql://c6401.ambari.apache.org:5432/druid', |
| 'druid.zk.service.host': ''}}, |
| 'druid-env': {'properties': {}, |
| 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'}, |
| 'druid.overlord.jvm.heap.memory': {'maximum': '49152'}, |
| 'druid.middlemanager.jvm.heap.memory': { |
| 'maximum': '49152'}, |
| 'druid.historical.jvm.heap.memory': {'maximum': '49152'}, |
| 'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}} |
| ) |
| |
| def test_recommendDruidConfigurations_WithDerby(self): |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 4, |
| "total_mem": 50331648, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| } |
| } |
| ] |
| } |
| |
| services = { |
| "Versions": { |
| "parent_stack_version": "2.5", |
| "stack_name": "HDP", |
| "stack_version": "2.6", |
| "stack_hierarchy": { |
| "stack_name": "HDP", |
| "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"] |
| } |
| }, |
| "services": [{ |
| "StackServices": { |
| "service_name": "DRUID", |
| }, |
| "components": [ |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_COORDINATOR", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_OVERLORD", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_BROKER", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_HISTORICAL", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_MIDDLEMANAGER", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| } |
| ] |
| } |
| ], |
| "configurations": { |
| "druid-common": { |
| "properties": { |
| "database_name": "druid", |
| "metastore_hostname": "c6401.ambari.apache.org", |
| "druid.metadata.storage.type": "derby", |
| "druid.extensions.loadList": "[\"mysql-metadata-storage\"]", |
| "druid.extensions.pullList": "[]" |
| } |
| } |
| } |
| } |
| |
| clusterData = { |
| } |
| |
| configurations = { |
| } |
| |
| self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, |
| {'druid-historical': { |
| 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, |
| 'druid-broker': { |
| 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}}, |
| 'druid-common': {'properties': {'druid.extensions.loadList': '[]', |
| 'druid.metadata.storage.connector.port': '1527', |
| 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true', |
| 'druid.zk.service.host': ''}}, |
| 'druid-env': {'properties': {}, |
| 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'}, |
| 'druid.overlord.jvm.heap.memory': {'maximum': '49152'}, |
| 'druid.middlemanager.jvm.heap.memory': { |
| 'maximum': '49152'}, |
| 'druid.historical.jvm.heap.memory': {'maximum': '49152'}, |
| 'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}} |
| ) |
| |
| |
| def test_recommendDruidConfigurations_property_existence_check(self): |
| # Test for https://issues.apache.org/jira/browse/AMBARI-19144 |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 4, |
| "total_mem": 50331648, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| } |
| } |
| ] |
| } |
| |
| services = { |
| "Versions": { |
| "parent_stack_version": "2.5", |
| "stack_name": "HDP", |
| "stack_version": "2.6", |
| "stack_hierarchy": { |
| "stack_name": "HDP", |
| "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"] |
| } |
| }, |
| "services": [{ |
| } |
| ], |
| "configurations": { |
| } |
| } |
| |
| clusterData = { |
| } |
| |
| configurations = { |
| } |
| |
| self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, |
| {} |
| ) |
| |
| def test_recommendDruidConfigurations_heterogeneous_hosts(self): |
| hosts = { |
| "items": [ |
| { |
| "href": "/api/v1/hosts/c6401.ambari.apache.org", |
| "Hosts": { |
| "cpu_count": 4, |
| "total_mem": 50331648, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| } |
| }, { |
| "href": "/api/v1/hosts/c6402.ambari.apache.org", |
| "Hosts": { |
| "cpu_count": 1, |
| "total_mem": 1922680, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6402.ambari.apache.org", |
| "host_name": "c6402.ambari.apache.org" |
| } |
| }, |
| { |
| "href": "/api/v1/hosts/c6403.ambari.apache.org", |
| "Hosts": { |
| "cpu_count": 3, |
| "total_mem": 3845360, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6403.ambari.apache.org", |
| "host_name": "c6403.ambari.apache.org" |
| } |
| } |
| ] |
| } |
| |
| services = { |
| "Versions": { |
| "parent_stack_version": "2.5", |
| "stack_name": "HDP", |
| "stack_version": "2.6", |
| "stack_hierarchy": { |
| "stack_name": "HDP", |
| "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"] |
| } |
| }, |
| "services": [{ |
| "StackServices": { |
| "service_name": "DRUID", |
| }, |
| "components": [ |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_COORDINATOR", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_OVERLORD", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_BROKER", |
| "hostnames": ["c6402.ambari.apache.org", "c6403.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_HISTORICAL", |
| "hostnames": ["c6401.ambari.apache.org", "c6403.ambari.apache.org"] |
| }, |
| }, |
| { |
| "StackServiceComponents": { |
| "component_name": "DRUID_MIDDLEMANAGER", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| } |
| ] |
| } |
| ], |
| "configurations": { |
| "druid-common": { |
| "properties": { |
| "database_name": "druid", |
| "metastore_hostname": "c6401.ambari.apache.org", |
| "druid.metadata.storage.type": "derby", |
| "druid.extensions.loadList": "[\"mysql-metadata-storage\"]", |
| "druid.extensions.pullList": "[]" |
| } |
| } |
| } |
| } |
| |
| clusterData = { |
| } |
| |
| configurations = { |
| } |
| |
| self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, |
| {'druid-historical': { |
| 'properties': {'druid.processing.numThreads': '2', 'druid.server.http.numThreads': '40'}}, |
| 'druid-broker': { |
| 'properties': {'druid.processing.numThreads': '1', 'druid.server.http.numThreads': '40'}}, |
| 'druid-common': {'properties': {'druid.extensions.loadList': '[]', |
| 'druid.metadata.storage.connector.port': '1527', |
| 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true', |
| 'druid.zk.service.host': '' |
| }}, |
| 'druid-env': {'properties': {}, |
| 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'}, |
| 'druid.overlord.jvm.heap.memory': {'maximum': '49152'}, |
| 'druid.middlemanager.jvm.heap.memory': { |
| 'maximum': '49152'}, |
| 'druid.historical.jvm.heap.memory': {'maximum': '3755'}, |
| 'druid.broker.jvm.heap.memory': {'maximum': '1877'}}}} |
| ) |
| |
| |
| def test_recommendAtlasConfigurations(self): |
| configurations = { |
| "application-properties": { |
| "properties": { |
| "atlas.sso.knox.providerurl": "", |
| "atlas.graph.index.search.solr.zookeeper-url": "", |
| "atlas.audit.hbase.zookeeper.quorum": "", |
| "atlas.graph.storage.hostname": "", |
| "atlas.kafka.bootstrap.servers": "", |
| "atlas.kafka.zookeeper.connect": "", |
| "atlas.authorizer.impl": "simple" |
| } |
| }, |
| "infra-solr-env": { |
| "properties": { |
| "infra_solr_znode": "/infra-solr" |
| } |
| }, |
| "ranger-atlas-plugin-properties": { |
| "properties": { |
| "ranger-atlas-plugin-enabled":"No" |
| } |
| }, |
| "atlas-env": { |
| "properties": { |
| "atlas_server_max_new_size": "600", |
| "atlas_server_xmx": "2048" |
| } |
| } |
| } |
| |
| clusterData = {} |
| |
| expected = { |
| "application-properties": { |
| "properties": { |
| "atlas.sso.knox.providerurl": "https://c6401.ambari.apache.org:8443/gateway/knoxsso/api/v1/websso", |
| "atlas.graph.index.search.solr.zookeeper-url": "", |
| "atlas.audit.hbase.zookeeper.quorum": "", |
| "atlas.graph.storage.hostname": "", |
| "atlas.kafka.bootstrap.servers": "", |
| "atlas.kafka.zookeeper.connect": "", |
| "atlas.authorizer.impl": "simple" |
| } |
| }, |
| "infra-solr-env": { |
| "properties": { |
| "infra_solr_znode": "/infra-solr" |
| } |
| }, |
| "ranger-atlas-plugin-properties": { |
| "properties": { |
| "ranger-atlas-plugin-enabled":"No" |
| } |
| }, |
| "atlas-env": { |
| "properties": { |
| "atlas_server_max_new_size": "600", |
| "atlas_server_xmx": "2048" |
| } |
| } |
| } |
| |
| services = { |
| "services": [ |
| { |
| "href": "/api/v1/stacks/HDP/versions/2.6/services/KNOX", |
| "StackServices": { |
| "service_name": "KNOX", |
| "service_version": "0.9.0.2.5", |
| "stack_name": "HDP", |
| "stack_version": "2.6" |
| }, |
| "components": [ |
| { |
| "href": "/api/v1/stacks/HDP/versions/2.6/services/KNOX/components/KNOX_GATEWAY", |
| "StackServiceComponents": { |
| "advertise_version": "false", |
| "cardinality": "1+", |
| "component_category": "MASTER", |
| "component_name": "KNOX_GATEWAY", |
| "display_name": "Knox Gateway", |
| "is_client": "false", |
| "is_master": "true", |
| "hostnames": ["c6401.ambari.apache.org"] |
| }, |
| "dependencies": [] |
| } |
| ] |
| } |
| ], |
| "configurations": configurations |
| } |
| |
| self.stackAdvisor.recommendAtlasConfigurations(configurations, clusterData, services, None) |
| self.assertEquals(configurations, expected) |
| |
| def test_recommendRangerConfigurations(self): |
| clusterData = {} |
| services = { |
| "Versions" : { |
| "stack_version" : "2.6", |
| }, |
| "services": [ |
| { |
| "StackServices": { |
| "service_name": "RANGER", |
| "service_version": "0.7.0.2.6" |
| }, |
| "components": [ |
| { |
| "StackServiceComponents": { |
| "component_name": "RANGER_ADMIN", |
| "hostnames": ["host1"] |
| } |
| } |
| ] |
| }, |
| ], |
| "configurations": { |
| "ranger-ugsync-site": { |
| "properties": { |
| "ranger.usersync.ldap.deltasync": "true", |
| "ranger.usersync.group.searchenabled": "false" |
| } |
| } |
| } |
| } |
| |
| expected = { |
| 'ranger-admin-site': { |
| 'properties': { |
| 'ranger.audit.solr.zookeepers': 'NONE', |
| 'ranger.audit.source.type': 'solr' |
| } |
| }, |
| 'admin-properties': { |
| 'properties': { |
| 'policymgr_external_url': 'http://host1:6080' |
| } |
| }, |
| 'ranger-tagsync-site': { |
| 'properties': {} |
| }, |
| 'tagsync-application-properties': { |
| 'properties': {} |
| }, |
| 'ranger-env': { |
| 'properties': { |
| 'ranger-storm-plugin-enabled': 'No' |
| } |
| }, |
| 'ranger-ugsync-site': { |
| 'properties': { |
| 'ranger.usersync.group.searchenabled': 'true' |
| } |
| } |
| } |
| |
| recommendedConfigurations = {} |
| |
| self.stackAdvisor.recommendRangerConfigurations(recommendedConfigurations, clusterData, services, None) |
| self.assertEquals(recommendedConfigurations, expected) |
| |
| def test_recommendRangerKMSConfigurations(self): |
| clusterData = {} |
| services = { |
| "ambari-server-properties": { |
| "ambari-server.user": "root" |
| }, |
| "Versions": { |
| "stack_version" : "2.6", |
| }, |
| "services": [ |
| { |
| "StackServices": { |
| "service_name": "RANGER_KMS", |
| "service_version": "0.7.0.2.6" |
| }, |
| "components": [ |
| { |
| "StackServiceComponents": { |
| "component_name": "RANGER_KMS_SERVER", |
| "hostnames": ["host1"] |
| } |
| } |
| ] |
| } |
| ], |
| "configurations": { |
| 'ranger-kms-site': { |
| 'properties': { |
| "ranger.service.https.attrib.ssl.enabled": "true", |
| "ranger.service.https.port": "9393" |
| } |
| } |
| } |
| } |
| |
| expected = { |
| 'kms-site': { |
| 'properties': {}, |
| 'property_attributes': { |
| 'hadoop.kms.proxyuser.HTTP.users': {'delete': 'true'}, |
| 'hadoop.kms.proxyuser.root.hosts': {'delete': 'true'}, |
| 'hadoop.kms.proxyuser.root.users': {'delete': 'true'}, |
| 'hadoop.kms.proxyuser.HTTP.hosts': {'delete': 'true'} |
| } |
| }, |
| 'core-site': { |
| 'properties': {} |
| }, |
| 'kms-properties': { |
| 'properties': {} |
| }, |
| 'ranger-kms-audit': { |
| 'properties': {} |
| }, |
| 'kms-env': { |
| 'properties': { |
| 'kms_port': '9393' |
| } |
| }, |
| 'dbks-site': { |
| 'properties': {} |
| } |
| } |
| |
| recommendedConfigurations = {} |
| |
| self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None) |
| self.assertEquals(recommendedConfigurations, expected) |
| |
| def test_recommendHDFSConfigurations(self): |
| ambariHostName = socket.getfqdn() |
| configurations = { |
| "ranger-hdfs-plugin-properties": { |
| "properties": { |
| "ranger-hdfs-plugin-enabled": "Yes", |
| "REPOSITORY_CONFIG_USERNAME":"hadoop" |
| } |
| }, |
| "hadoop-env":{ |
| "properties":{ |
| "hdfs_user":"custom_hdfs" |
| } |
| } |
| } |
| clusterData = { |
| "totalAvailableRam": 2048, |
| "hBaseInstalled": True, |
| "hbaseRam": 112, |
| "reservedRam": 128 |
| } |
| |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "disk_info": [{ |
| "size": '8', |
| "mountpoint": "/" |
| }] |
| } |
| }]} |
| |
| services = { |
| "services": |
| [ |
| { |
| "StackServices": { |
| "service_name" : "HDFS", |
| "service_version" : "2.7.0.2.6" |
| }, |
| "components": [ |
| ] |
| } |
| ], |
| "Versions": { |
| "stack_version": "2.6" |
| }, |
| "configurations": configurations, |
| "ambari-server-properties": {"ambari-server.user":"ambari_user"} |
| } |
| |
| |
| expected = { |
| 'core-site': { |
| 'properties': { |
| 'hadoop.proxyuser.ambari_user.groups': '*', |
| 'hadoop.proxyuser.custom_hdfs.groups': '*', |
| 'hadoop.proxyuser.custom_hdfs.hosts': '*', |
| 'hadoop.proxyuser.ambari_user.hosts': ambariHostName |
| }, |
| 'property_attributes': { |
| 'hadoop.security.key.provider.path': { |
| 'delete': 'true' |
| } |
| } |
| }, |
| 'hadoop-env': { |
| 'properties': { |
| 'hdfs_user': 'custom_hdfs', |
| 'namenode_heapsize': '1024', |
| 'namenode_opt_maxnewsize': '128', |
| 'namenode_opt_newsize': '128' |
| } |
| }, |
| 'hdfs-site': { |
| 'properties': { |
| 'dfs.datanode.data.dir': '/hadoop/hdfs/data', |
| 'dfs.datanode.failed.volumes.tolerated': '0', |
| 'dfs.datanode.max.transfer.threads': '16384', |
| 'dfs.namenode.name.dir': '/hadoop/hdfs/namenode', |
| 'dfs.namenode.handler.count': '100', |
| 'dfs.namenode.checkpoint.dir': '/hadoop/hdfs/namesecondary', |
| 'dfs.namenode.safemode.threshold-pct': '1.000', |
| 'dfs.namenode.inode.attributes.provider.class': 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer', |
| 'dfs.datanode.du.reserved': '1073741824' |
| }, |
| 'property_attributes': { |
| 'dfs.datanode.failed.volumes.tolerated': { |
| 'maximum': '1' |
| }, |
| 'dfs.encryption.key.provider.uri': { |
| 'delete': 'true' |
| } |
| } |
| }, |
| 'ranger-hdfs-plugin-properties': { |
| 'properties': { |
| 'ranger-hdfs-plugin-enabled': 'Yes', |
| 'REPOSITORY_CONFIG_USERNAME': 'custom_hdfs' |
| } |
| } |
| } |
| |
| self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations,expected) |
| configurations['hadoop-env']['properties']['hdfs_user'] = 'hadoop' |
| expected['hadoop-env']['properties']['hdfs_user'] = 'hadoop' |
| expected['ranger-hdfs-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'hadoop' |
| expected['core-site']['properties']['hadoop.proxyuser.hadoop.hosts'] = '*' |
| expected['core-site']['properties']['hadoop.proxyuser.hadoop.groups'] = '*' |
| self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations,expected) |
| |
| |
| def test_recommendHiveConfigurations(self): |
| configurations = { |
| "ranger-hive-plugin-properties": { |
| "properties": { |
| "ranger-hive-plugin-enabled": "Yes", |
| "REPOSITORY_CONFIG_USERNAME":"hive" |
| } |
| }, |
| "hive-env":{ |
| "properties":{ |
| "hive_security_authorization":"ranger", |
| "hive_user":"custom_hive" |
| } |
| } |
| } |
| clusterData = { |
| "cpu": 4, |
| "mapMemory": 3000, |
| "amMemory": 2000, |
| "reduceMemory": 2056, |
| "containers": 3, |
| "ramPerContainer": 256, |
| "yarnMinContainerSize": 256 |
| } |
| |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 6, |
| "total_mem": 50331648, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| }, |
| } |
| ] |
| } |
| |
| services = { |
| "services": |
| [{ |
| "StackServices": { |
| "service_name": "YARN" |
| }, "components": [] |
| }, |
| { |
| "StackServices": { |
| "service_name" : "HIVE", |
| "service_version" : "1.2.1.2.6" |
| }, |
| "components": [ |
| ] |
| } |
| ], |
| "Versions": { |
| "stack_name" : "HDP", |
| "stack_version": "2.6" |
| }, |
| "changed-configurations": [ |
| ], |
| "configurations": configurations, |
| "ambari-server-properties": {"ambari-server.user":"ambari_user"} |
| } |
| |
| |
| expected = { |
| 'yarn-env': { |
| 'properties': { |
| 'min_user_id': '500', |
| 'apptimelineserver_heapsize': '8072', |
| 'service_check.queue.name': 'default' |
| } |
| }, |
| 'ranger-hive-plugin-properties': { |
| 'properties': { |
| 'ranger-hive-plugin-enabled': 'Yes', |
| 'REPOSITORY_CONFIG_USERNAME': 'custom_hive' |
| } |
| }, |
| 'webhcat-site': { |
| 'properties': { |
| 'templeton.hadoop.queue.name': 'default' |
| } |
| }, |
| 'hive-interactive-env': { |
| 'properties': { |
| 'enable_hive_interactive': 'false' |
| }, |
| 'property_attributes': { |
| 'num_llap_nodes': { |
| 'read_only': 'true' |
| } |
| } |
| }, |
| 'hive-env': { |
| 'properties': { |
| 'hive.atlas.hook': 'false', |
| 'hive_security_authorization': 'ranger', |
| 'hive_exec_orc_storage_strategy': 'SPEED', |
| 'hive_timeline_logging_enabled': 'true', |
| 'hive_txn_acid': 'off', |
| 'hive_user': 'custom_hive' |
| } |
| }, |
| 'hiveserver2-site': { |
| 'properties': { |
| 'hive.security.authorization.enabled': 'true', |
| 'hive.conf.restricted.list': 'hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager,hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled', |
| 'hive.security.authenticator.manager': 'org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator', |
| 'hive.security.authorization.manager': 'org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory' |
| } |
| }, |
| 'hive-site': { |
| 'properties': { |
| 'hive.tez.container.size': '768', |
| 'hive.exec.orc.default.stripe.size': '67108864', |
| 'hive.execution.engine': 'mr', |
| 'hive.vectorized.execution.reduce.enabled': 'false', |
| 'hive.compactor.worker.threads': '0', |
| 'hive.compactor.initiator.on': 'false', |
| 'hive.exec.pre.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', |
| 'hive.compute.query.using.stats': 'true', |
| 'hive.exec.orc.default.compress': 'ZLIB', |
| 'hive.exec.orc.encoding.strategy': 'SPEED', |
| 'hive.server2.tez.initialize.default.sessions': 'false', |
| 'hive.security.authorization.enabled': 'true', |
| 'hive.exec.post.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', |
| 'hive.server2.tez.default.queues': 'default', |
| 'hive.prewarm.enabled': 'false', |
| 'hive.exec.orc.compression.strategy': 'SPEED', |
| 'hive.optimize.index.filter': 'true', |
| 'hive.auto.convert.join.noconditionaltask.size': '214748364', |
| 'hive.vectorized.execution.enabled': 'true', |
| 'hive.exec.reducers.bytes.per.reducer': '67108864', |
| 'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager', |
| 'hive.server2.tez.sessions.per.default.queue': '1', |
| 'hive.prewarm.numcontainers': '3', |
| 'hive.tez.dynamic.partition.pruning': 'true', |
| 'hive.tez.auto.reducer.parallelism': 'true', |
| 'hive.server2.use.SSL': 'false', |
| 'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', |
| 'hive.support.concurrency': 'false', |
| 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps', |
| 'hive.security.metastore.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider', |
| 'hive.exec.dynamic.partition.mode': 'strict', |
| 'hive.optimize.sort.dynamic.partition': 'false', |
| 'hive.server2.enable.doAs': 'false' |
| }, |
| 'property_attributes': { |
| 'hive.tez.container.size': { |
| 'minimum': '256', |
| 'maximum': '768' |
| }, |
| 'atlas.cluster.name': { |
| 'delete': 'true' |
| }, |
| 'hive.server2.tez.default.queues': { |
| 'entries': [ |
| { |
| 'value': 'default', |
| 'label': 'default queue' |
| } |
| ] |
| }, |
| 'datanucleus.rdbms.datastoreAdapterClassName': { |
| 'delete': 'true' |
| }, |
| 'hive.auto.convert.join.noconditionaltask.size': { |
| 'maximum': '644245094' |
| }, |
| 'atlas.rest.address': { |
| 'delete': 'true' |
| } |
| } |
| }, |
| 'hive-interactive-site': { |
| 'properties': {} |
| }, |
| 'yarn-site': { |
| 'properties': { |
| 'hadoop.registry.rm.enabled': 'false', |
| 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes': '', |
| 'yarn.scheduler.minimum-allocation-vcores': '1', |
| 'yarn.scheduler.maximum-allocation-vcores': '4', |
| 'yarn.nodemanager.resource.memory-mb': '768', |
| 'yarn.nodemanager.local-dirs': '/hadoop/yarn/local,/dev/shm/hadoop/yarn/local,/vagrant/hadoop/yarn/local', |
| 'yarn.nodemanager.log-dirs': '/hadoop/yarn/log,/dev/shm/hadoop/yarn/log,/vagrant/hadoop/yarn/log', |
| 'yarn.timeline-service.entity-group-fs-store.app-cache-size': '10', |
| 'yarn.scheduler.minimum-allocation-mb': '256', |
| 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '', |
| 'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round': '1.0', |
| 'yarn.nodemanager.resource.cpu-vcores': '4', |
| 'yarn.scheduler.maximum-allocation-mb': '768', |
| 'yarn.nodemanager.linux-container-executor.group': 'hadoop', |
| 'yarn.timeline-service.leveldb-state-store.path': '/hadoop/yarn/timeline', |
| 'yarn.timeline-service.leveldb-timeline-store.path': '/hadoop/yarn/timeline' |
| }, |
| 'property_attributes': { |
| 'yarn.authorization-provider': { |
| 'delete': 'true' |
| } |
| } |
| } |
| } |
| |
| self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations,expected) |
| configurations['hive-env']['properties']['hive_user'] = 'hive' |
| expected['hive-env']['properties']['hive_user'] = 'hive' |
| expected['ranger-hive-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'hive' |
| self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations,expected) |
| |
| |
| def test_recommendHBASEConfigurations(self): |
| configurations = { |
| "ranger-hbase-plugin-properties": { |
| "properties": { |
| "ranger-hbase-plugin-enabled": "Yes", |
| "REPOSITORY_CONFIG_USERNAME":"hbase" |
| } |
| }, |
| "hbase-env":{ |
| "properties":{ |
| "hbase_user":"custom_hbase" |
| } |
| } |
| } |
| |
| services = { |
| "services": [{ |
| "StackServices": { |
| "service_name" : "HBASE", |
| "service_version" : "1.1.2.2.6" |
| }, |
| "components": [ |
| ] |
| } |
| ], |
| "Versions": { |
| "stack_name" : "HDP", |
| "stack_version": "2.6" |
| }, |
| "configurations": configurations, |
| "ambari-server-properties": {"ambari-server.user":"ambari_user"} |
| } |
| |
| |
| clusterData = { |
| "totalAvailableRam": 2048, |
| "hBaseInstalled": True, |
| "hbaseRam": 112, |
| "reservedRam": 128 |
| } |
| expected = { |
| 'hbase-site': { |
| 'properties': { |
| 'hbase.regionserver.wal.codec': 'org.apache.hadoop.hbase.regionserver.wal.WALCellCodec', |
| 'hbase.master.ui.readonly': 'false', |
| 'hbase.security.authorization': 'true', |
| 'hbase.bucketcache.percentage.in.combinedcache': '1.0000', |
| 'hbase.regionserver.global.memstore.size': '0.4', |
| 'hfile.block.cache.size': '0.4', |
| 'hbase.coprocessor.region.classes': 'org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor', |
| 'hbase.bucketcache.size': '92160', |
| 'hbase.coprocessor.regionserver.classes': 'org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor', |
| 'hbase.coprocessor.master.classes': 'org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor', |
| 'hbase.bucketcache.ioengine': 'offheap' |
| }, |
| 'property_attributes': { |
| 'hbase.bucketcache.percentage.in.combinedcache': { |
| 'delete': 'true' |
| }, |
| 'hbase.region.server.rpc.scheduler.factory.class': { |
| 'delete': 'true' |
| } |
| } |
| }, |
| 'ranger-hbase-plugin-properties': { |
| 'properties': { |
| 'REPOSITORY_CONFIG_USERNAME': 'custom_hbase', |
| 'ranger-hbase-plugin-enabled': 'Yes' |
| } |
| }, |
| 'hbase-env': { |
| 'properties': { |
| 'hbase_user': 'custom_hbase', |
| 'hbase_master_heapsize': '1024', |
| 'hbase_regionserver_heapsize': '20480', |
| 'hbase_max_direct_memory_size': '94208' |
| } |
| }, |
| 'core-site': { |
| 'properties': {} |
| } |
| } |
| self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services, None) |
| self.assertEquals(configurations, expected) |
| configurations['hbase-env']['properties']['hbase_user'] = 'hbase' |
| expected['hbase-env']['properties']['hbase_user'] = 'hbase' |
| expected['ranger-hbase-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'hbase' |
| self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services, None) |
| self.assertEquals(configurations, expected) |
| |
| def test_recommendYARNConfigurations(self): |
| configurations = { |
| "yarn-env": { |
| "properties": { |
| "yarn_user" : "custom_yarn" |
| } |
| }, |
| "ranger-yarn-plugin-properties": { |
| "properties": { |
| "ranger-yarn-plugin-enabled" : "Yes", |
| "REPOSITORY_CONFIG_USERNAME":"yarn" |
| } |
| } |
| } |
| services = { |
| "services" : [{ |
| "StackServices": { |
| "service_name" : "YARN", |
| "service_version" : "2.7.3.2.6" |
| }, |
| "components": [] |
| } |
| ], |
| "changed-configurations": [ |
| ], |
| "configurations": configurations |
| } |
| |
| |
| clusterData = { |
| "cpu": 4, |
| "containers" : 5, |
| "ramPerContainer": 256, |
| "yarnMinContainerSize": 256 |
| } |
| expected = { |
| 'yarn-env': { |
| 'properties': { |
| 'yarn_user': 'custom_yarn', |
| 'service_check.queue.name': 'default', |
| 'min_user_id': '500', |
| 'apptimelineserver_heapsize': '2048' |
| } |
| }, |
| 'ranger-yarn-plugin-properties': { |
| 'properties': { |
| 'ranger-yarn-plugin-enabled': 'Yes', |
| 'REPOSITORY_CONFIG_USERNAME': 'custom_yarn' |
| } |
| }, |
| 'yarn-site': { |
| 'properties': { |
| 'hadoop.registry.rm.enabled': 'false', |
| 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes': '', |
| 'yarn.authorization-provider': 'org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer', |
| 'yarn.acl.enable': 'true', |
| 'yarn.scheduler.minimum-allocation-vcores': '1', |
| 'yarn.scheduler.maximum-allocation-vcores': '4', |
| 'yarn.nodemanager.resource.memory-mb': '1280', |
| 'yarn.scheduler.minimum-allocation-mb': '256', |
| 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '', |
| 'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round': '1.0', |
| 'yarn.nodemanager.resource.cpu-vcores': '4', |
| 'yarn.scheduler.maximum-allocation-mb': '1280', |
| 'yarn.nodemanager.linux-container-executor.group': 'hadoop', |
| 'yarn.nodemanager.local-dirs': '/hadoop/yarn/local,/dev/shm/hadoop/yarn/local,/vagrant/hadoop/yarn/local', |
| 'yarn.nodemanager.log-dirs': '/hadoop/yarn/log,/dev/shm/hadoop/yarn/log,/vagrant/hadoop/yarn/log', |
| 'yarn.timeline-service.entity-group-fs-store.app-cache-size': '7', |
| 'yarn.timeline-service.leveldb-state-store.path': '/hadoop/yarn/timeline', |
| 'yarn.timeline-service.leveldb-timeline-store.path': '/hadoop/yarn/timeline' |
| |
| } |
| } |
| } |
| |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 6, |
| "total_mem": 4096, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| }, |
| } |
| ] |
| } |
| |
| self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, expected) |
| |
| configurations['yarn-env']['properties']['yarn_user'] = 'yarn' |
| expected['yarn-env']['properties']['yarn_user'] = 'yarn' |
| expected['ranger-yarn-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'yarn' |
| self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, expected) |
| |
| |
| |
| def test_recommendYARNConfigurations_for_ats_heapsize_and_cache(self): |
| configurations = { |
| "yarn-env": { |
| "properties": { |
| "yarn_user" : "custom_yarn" |
| } |
| }, |
| "ranger-yarn-plugin-properties": { |
| "properties": { |
| "ranger-yarn-plugin-enabled" : "Yes", |
| "REPOSITORY_CONFIG_USERNAME":"yarn" |
| } |
| } |
| } |
| services = { |
| "services" : [{ |
| "StackServices": { |
| "service_name" : "YARN", |
| "service_version" : "2.7.3.2.6" |
| }, |
| "components": [] |
| } |
| ], |
| "changed-configurations": [ |
| ], |
| "configurations": configurations |
| } |
| |
| |
| clusterData = { |
| "cpu": 4, |
| "containers" : 5, |
| "ramPerContainer": 256, |
| "yarnMinContainerSize": 256 |
| } |
| expected = { |
| 'yarn-env': { |
| 'properties': { |
| 'yarn_user': 'custom_yarn', |
| 'service_check.queue.name': 'default', |
| 'min_user_id': '500', |
| 'apptimelineserver_heapsize': '1024' |
| } |
| }, |
| 'ranger-yarn-plugin-properties': { |
| 'properties': { |
| 'ranger-yarn-plugin-enabled': 'Yes', |
| 'REPOSITORY_CONFIG_USERNAME': 'custom_yarn' |
| } |
| }, |
| 'yarn-site': { |
| 'properties': { |
| 'hadoop.registry.rm.enabled': 'false', |
| 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes': '', |
| 'yarn.authorization-provider': 'org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer', |
| 'yarn.acl.enable': 'true', |
| 'yarn.scheduler.minimum-allocation-vcores': '1', |
| 'yarn.scheduler.maximum-allocation-vcores': '4', |
| 'yarn.nodemanager.resource.memory-mb': '1280', |
| 'yarn.scheduler.minimum-allocation-mb': '256', |
| 'yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath': '', |
| 'yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round': '1.0', |
| 'yarn.nodemanager.resource.cpu-vcores': '4', |
| 'yarn.scheduler.maximum-allocation-mb': '1280', |
| 'yarn.nodemanager.linux-container-executor.group': 'hadoop', |
| 'yarn.nodemanager.local-dirs': '/hadoop/yarn/local,/dev/shm/hadoop/yarn/local,/vagrant/hadoop/yarn/local', |
| 'yarn.nodemanager.log-dirs': '/hadoop/yarn/log,/dev/shm/hadoop/yarn/log,/vagrant/hadoop/yarn/log', |
| 'yarn.timeline-service.entity-group-fs-store.app-cache-size': '3', |
| 'yarn.timeline-service.leveldb-state-store.path': '/hadoop/yarn/timeline', |
| 'yarn.timeline-service.leveldb-timeline-store.path': '/hadoop/yarn/timeline' |
| |
| } |
| } |
| } |
| |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 6, |
| "total_mem": 2048, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| }, |
| } |
| ] |
| } |
| |
| |
| |
| |
| ''' |
| Test 1 : |
| I/P: |
| - 'changed-configurations' is empty (doesnt have 'yarn.timeline-service.entity-group-fs-store.app-cache-size') |
| - 'host_mem' = 2048 |
| O/P : |
| - Config value recommended for: |
| - yarn.timeline-service.entity-group-fs-store.app-cache-size = 3 |
| - apptimelineserver_heapsize = 1024 |
| ''' |
| |
| self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, expected) |
| |
| |
| |
| ''' |
| Test 2 : |
| I/P: |
| - 'changed-configurations' is empty (doesnt have 'yarn.timeline-service.entity-group-fs-store.app-cache-size') |
| - 'host_mem' = 4096 |
| O/P : |
| - Config value recommended for: |
| - yarn.timeline-service.entity-group-fs-store.app-cache-size = 7 |
| - apptimelineserver_heapsize = 2048 |
| ''' |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 6, |
| "total_mem": 4096, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| }, |
| } |
| ] |
| } |
| |
| expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '2048' |
| expected['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.app-cache-size'] = '7' |
| self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, expected) |
| |
| |
| |
| ''' |
| Test 3 : |
| I/P: |
| - 'changed-configurations' is empty (doesnt have 'yarn.timeline-service.entity-group-fs-store.app-cache-size') |
| - 'host_mem' = 8192 |
| O/P : |
| - Config value recommended for: |
| - yarn.timeline-service.entity-group-fs-store.app-cache-size = 10 |
| - apptimelineserver_heapsize = 4096 |
| ''' |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 6, |
| "total_mem": 8192, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| }, |
| } |
| ] |
| } |
| |
| expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '4096' |
| expected['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.app-cache-size'] = '10' |
| self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, expected) |
| |
| |
| |
| ''' |
| Test 4 : |
| I/P: |
| - 'changed-configurations' has 'yarn.timeline-service.entity-group-fs-store.app-cache-size' |
| - 'host_mem' = 2048 |
| O/P : |
| - Config value recommended for: |
| - apptimelineserver_heapsize = 4096 |
| ''' |
| |
| services["changed-configurations"] = [ |
| { |
| u'old_value': u'10', |
| u'type': u'yarn-site', |
| u'name': u'yarn.timeline-service.entity-group-fs-store.app-cache-size' |
| } |
| ] |
| |
| services["configurations"] = { |
| "yarn-env": { |
| "properties": { |
| "yarn_user" : "custom_yarn", |
| } |
| }, |
| "yarn-site": { |
| "properties": { |
| "yarn.timeline-service.entity-group-fs-store.app-cache-size" : "7" |
| } |
| }, |
| "ranger-yarn-plugin-properties": { |
| "properties": { |
| "ranger-yarn-plugin-enabled" : "Yes", |
| "REPOSITORY_CONFIG_USERNAME":"yarn" |
| } |
| } |
| } |
| |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 6, |
| "total_mem": 4096, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| }, |
| } |
| ] |
| } |
| |
| |
| |
| ''' |
| Test 5 : |
| I/P: |
| - 'changed-configurations' has 'yarn.timeline-service.entity-group-fs-store.app-cache-size' |
| - 'host_mem' = 4096 |
| O/P : |
| - Config value recommended for: |
| - apptimelineserver_heapsize = 2048 |
| ''' |
| |
| services["changed-configurations"] = [ |
| { |
| u'old_value': u'10', |
| u'type': u'yarn-site', |
| u'name': u'yarn.timeline-service.entity-group-fs-store.app-cache-size' |
| } |
| ] |
| |
| services["configurations"] = { |
| "yarn-env": { |
| "properties": { |
| "yarn_user" : "custom_yarn", |
| } |
| }, |
| "yarn-site": { |
| "properties": { |
| "yarn.timeline-service.entity-group-fs-store.app-cache-size" : "7" |
| } |
| }, |
| "ranger-yarn-plugin-properties": { |
| "properties": { |
| "ranger-yarn-plugin-enabled" : "Yes", |
| "REPOSITORY_CONFIG_USERNAME":"yarn" |
| } |
| } |
| } |
| |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 6, |
| "total_mem": 4096, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| }, |
| } |
| ] |
| } |
| |
| |
| expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '2048' |
| self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, expected) |
| |
| |
| |
| ''' |
| Test 6 : |
| I/P: |
| - 'changed-configurations' has 'yarn.timeline-service.entity-group-fs-store.app-cache-size' |
| - 'host_mem' = 8196 |
| O/P : |
| - Config value recommended for: |
| - Shouldn't have yarn.timeline-service.entity-group-fs-store.app-cache-size |
| - apptimelineserver_heapsize = 4572 |
| ''' |
| |
| services["changed-configurations"] = [ |
| { |
| u'old_value': u'10', |
| u'type': u'yarn-site', |
| u'name': u'yarn.timeline-service.entity-group-fs-store.app-cache-size' |
| } |
| ] |
| |
| services["configurations"] = { |
| "yarn-env": { |
| "properties": { |
| "yarn_user" : "custom_yarn", |
| } |
| }, |
| "yarn-site": { |
| "properties": { |
| "yarn.timeline-service.entity-group-fs-store.app-cache-size" : "3" |
| } |
| }, |
| "ranger-yarn-plugin-properties": { |
| "properties": { |
| "ranger-yarn-plugin-enabled" : "Yes", |
| "REPOSITORY_CONFIG_USERNAME":"yarn" |
| } |
| } |
| } |
| |
| hosts = { |
| "items": [ |
| { |
| "Hosts": { |
| "cpu_count": 6, |
| "total_mem": 16392, |
| "disk_info": [ |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"}, |
| {"mountpoint": "/"}, |
| {"mountpoint": "/dev/shm"}, |
| {"mountpoint": "/vagrant"} |
| ], |
| "public_host_name": "c6401.ambari.apache.org", |
| "host_name": "c6401.ambari.apache.org" |
| }, |
| } |
| ] |
| } |
| |
| |
| expected['yarn-env']['properties']['apptimelineserver_heapsize'] = '4572' |
| self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) |
| self.assertEquals(configurations, expected) |
| |
| |
| def test_recommendKAFKAConfigurations(self): |
| configurations = { |
| "kafka-env": { |
| "properties": { |
| "kafka_user" : "custom_kafka" |
| } |
| }, |
| "ranger-kafka-plugin-properties": { |
| "properties": { |
| "ranger-kafka-plugin-enabled" : "Yes", |
| "REPOSITORY_CONFIG_USERNAME":"kafka" |
| } |
| } |
| } |
| clusterData = [] |
| services = { |
| "services" : [{ |
| "StackServices": { |
| "service_name" : "KAFKA", |
| "service_version" : "0.10.0.2.6" |
| }, |
| "components": [] |
| } |
| ], |
| "configurations": configurations |
| } |
| |
| expected = { |
| 'kafka-env': { |
| 'properties': { |
| 'kafka_user': 'custom_kafka' |
| } |
| }, |
| 'kafka-log4j': { |
| 'properties': {} |
| }, |
| 'kafka-broker': { |
| 'properties': {}, |
| 'property_attributes': { |
| 'principal.to.local.class': { |
| 'delete': 'true' |
| }, |
| 'super.users': { |
| 'delete': 'true' |
| }, |
| 'security.inter.broker.protocol': { |
| 'delete': 'true' |
| }, |
| 'authorizer.class.name': { |
| 'delete': 'true' |
| } |
| } |
| }, |
| 'ranger-kafka-plugin-properties': { |
| 'properties': { |
| 'ranger-kafka-plugin-enabled': 'Yes', |
| 'REPOSITORY_CONFIG_USERNAME': 'custom_kafka' |
| } |
| } |
| } |
| |
| self.stackAdvisor.recommendKAFKAConfigurations(configurations, clusterData, services, None) |
| self.assertEquals(configurations, expected) |
| configurations['kafka-env']['properties']['kafka_user'] = 'kafka' |
| expected['kafka-env']['properties']['kafka_user'] = 'kafka' |
| expected['ranger-kafka-plugin-properties']['properties']['REPOSITORY_CONFIG_USERNAME'] = 'kafka' |
| self.stackAdvisor.recommendKAFKAConfigurations(configurations, clusterData, services, None) |
| self.assertEquals(configurations, expected) |
| |
| def load_json(self, filename): |
| file = os.path.join(self.testDirectory, filename) |
| with open(file, 'rb') as f: |
| data = json.load(f) |
| return data |
