LENS-1389: Back Merge with master
diff --git a/contrib/clients/python/lens/client/main.py b/contrib/clients/python/lens/client/main.py
index bf5d81e..b1846ad 100644
--- a/contrib/clients/python/lens/client/main.py
+++ b/contrib/clients/python/lens/client/main.py
@@ -21,10 +21,12 @@
from .session import LensSessionClient
from .query import LensQueryClient
from .utils import xml_file_to_conf
-
+import logging
+logger = logging.getLogger(__name__)
class LensClient(object):
- def __init__(self, base_url=None, username="", password="", database=None, conf=None):
+ def __init__(self, base_url=None, username="", password="", database=None, conf=None, logging_level=logging.INFO):
+ logging.basicConfig(level=logging_level)
if conf and isinstance(conf, string_types) and os.path.exists(conf):
if os.path.isdir(conf):
conf = os.path.join(conf, 'lens-client-site.xml')
@@ -35,6 +37,7 @@
self.base_url = base_url or conf.get('lens.server.base.url', "http://0.0.0.0:9999/lensapi")
if self.base_url[-1] != '/':
self.base_url += "/"
+ logger.debug("Using conf %s", conf)
username = username or conf.get('lens.client.user.name', "anonymous")
database = database or conf.get('lens.client.dbname')
self.session = LensSessionClient(self.base_url, username, password, database, conf)
diff --git a/contrib/clients/python/lens/client/query.py b/contrib/clients/python/lens/client/query.py
index f82f0cb..df16cca 100644
--- a/contrib/clients/python/lens/client/query.py
+++ b/contrib/clients/python/lens/client/query.py
@@ -15,21 +15,26 @@
# limitations under the License.
#
import codecs
+import csv
+import logging
import time
import zipfile
import requests
-from six import string_types, BytesIO, StringIO, PY2, PY3
+from requests.exceptions import HTTPError
+from six import string_types, BytesIO, PY2, PY3
+
from .models import WrappedJson
from .utils import conf_to_xml
-import csv
+logger = logging.getLogger(__name__)
long_type = int
if PY3:
from collections.abc import Iterable as Iterable
elif PY2:
from collections import Iterable as Iterable
+
long_type = long
@@ -70,6 +75,7 @@
}
default_mapping = lambda x: x
+
class LensQueryResult(Iterable):
def __init__(self, custom_mappings=None):
if custom_mappings is None:
@@ -93,6 +99,7 @@
for row in self.rows:
yield list(self._mapping(value.type)(value.value) if value else None for value in row['values'])
+
class LensPersistentResult(LensQueryResult):
def __init__(self, header, response, encoding=None, is_header_present=True, delimiter=",",
custom_mappings=None):
@@ -107,6 +114,11 @@
def _parse_line(self, line):
return list(self._mapping(self.header.columns[index].type)(line[index]) for index in range(len(line)))
+ def get_csv_reader(self, file):
+ if PY3:
+ file = codecs.iterdecode(file, 'utf-8')
+ return csv.reader(file, delimiter=self.delimiter)
+
def __iter__(self):
if self.is_zipped:
byte_stream = BytesIO(self.response.content)
@@ -114,7 +126,7 @@
for name in self.zipfile.namelist():
with self.zipfile.open(name) as single_file:
if name[-3:] == 'csv':
- reader = csv.reader(single_file, delimiter=self.delimiter)
+ reader = self.get_csv_reader(single_file)
else:
reader = single_file
reader_iterator = iter(reader)
@@ -142,7 +154,8 @@
self.launched_queries = []
self.finished_queries = {}
self.query_confs = {}
- self.is_header_present_in_result = self._session['lens.query.output.write.header'].lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup']
+ self.is_header_present_in_result = self._session['lens.query.output.write.header'].lower() \
+ in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup']
def __call__(self, **filters):
filters['sessionid'] = self._session._sessionid
@@ -183,6 +196,7 @@
payload.append(('conf', conf_to_xml(conf)))
resp = requests.post(self.base_url + "queries/", files=payload, headers={'accept': 'application/json'})
query = self.sanitize_response(resp)
+ logger.info("Submitted query %s", query)
if conf:
self.query_confs[str(query)] = conf
if fetch_result:
@@ -216,7 +230,8 @@
return LensPersistentResult(metadata, resp, is_header_present=is_header_present, *args, **kwargs)
else:
response = requests.get(self.base_url + "queries/" + handle + "/resultset",
- params={'sessionid': self._session._sessionid}, headers={'accept': 'application/json'})
+ params={'sessionid': self._session._sessionid},
+ headers={'accept': 'application/json'})
resp = self.sanitize_response(response)
# If it has in memory result, return inmemory result iterator
if resp._is_wrapper and resp._wrapped_key == u'inMemoryQueryResult':
@@ -228,13 +243,18 @@
raise Exception("Result set not available")
def sanitize_response(self, resp):
- resp.raise_for_status()
try:
resp_json = resp.json(object_hook=WrappedJson)
+ except:
+ resp_json = resp.json()
+ if resp_json is not None:
if 'lensAPIResult' in resp_json:
resp_json = resp_json.lens_a_p_i_result
if 'error' in resp_json:
- raise Exception(resp_json['error'])
+ error = resp_json['error']
+ if "stackTrace" in error:
+ logger.error(error['stackTrace'])
+ raise HTTPError(error, request=resp.request, response=resp)
if 'data' in resp_json:
data = resp_json.data
if len(data) == 2 and 'type' in data:
@@ -242,6 +262,9 @@
keys.remove('type')
return WrappedJson({data['type']: data[keys[0]]})
return data
- except:
- resp_json = resp.json()
- return resp_json
+ if resp_json is not None:
+ return resp_json
+ else:
+ resp.raise_for_status()
+ logger.error(resp.text)
+ raise Exception("Unknown error with response", resp)
diff --git a/contrib/clients/python/setup.py b/contrib/clients/python/setup.py
index 113205d..de59d32 100644
--- a/contrib/clients/python/setup.py
+++ b/contrib/clients/python/setup.py
@@ -45,6 +45,8 @@
def run_tests(self):
#import here, cause outside the eggs aren't loaded
import tox
+ import os
+ del os.environ["PYTHONPATH"]
errcode = tox.cmdline(self.test_args)
sys.exit(errcode)
diff --git a/contrib/clients/python/test/test_lensclient.py b/contrib/clients/python/test/test_lensclient.py
index e9b0bdb..b8d7c73 100644
--- a/contrib/clients/python/test/test_lensclient.py
+++ b/contrib/clients/python/test/test_lensclient.py
@@ -29,7 +29,9 @@
from requests.exceptions import HTTPError
from lens.client import LensClient
-
+import logging
+logger = logging.getLogger(__name__)
+logging.basicConfig(level="DEBUG")
def check_output(command):
output = subprocess.check_output(command.split())
@@ -85,11 +87,14 @@
cls.base_path = glob.glob(joined)[0]
with cwd(cls.base_path):
with cwd('server'):
+ logger.info("Inside server directory")
server_start_output = check_output("bin/lens-ctl restart")
assert "Started lens server" in server_start_output
assert os.path.exists('logs/server.pid')
+ logger.info("started lens server")
time.sleep(1)
while not os.path.exists('logs/lensserver.log'):
+ logger.info("waiting for lensserver.log to be created")
error = get_error()
if has_error(error):
# Assert again with complete error
@@ -97,9 +102,11 @@
time.sleep(1)
error = get_error()
if has_error(error):
+ logger.error(error)
assert False, error
-
+ logger.info("finished setting up server environment. Will setup client now")
with cwd('client'):
+ logger.info("Inside client directory")
cls.candidate_query = select_query('examples/resources/cube-queries.sql')
with open('check_connection.sql', 'w') as f:
f.write('show databases')
@@ -160,7 +167,7 @@
handle = client.queries.submit(self.candidate_query, query_name="Candidate Query")
finished_query = client.queries.wait_till_finish(handle)
assert client.queries[handle] == finished_query
- queries = client.queries(state='SUCCESSFUL', fromDate=finished_query.submission_time - 1,
+ queries = client.queries(state='successful,failed', fromDate=finished_query.submission_time - 1,
toDate=finished_query.submission_time + 1)
assert handle in queries
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 7608a43..b445447 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -307,6 +307,17 @@
}
}
+ public void createCubeFactTable(String cubeName, String factName, List<FieldSchema> columns,
+ Map<String, Set<UpdatePeriod>> storageAggregatePeriods, double weight, Map<String, String> properties,
+ Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap)
+ throws LensException {
+ CubeFactTable factTable = new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight,
+ properties, storageUpdatePeriodMap);
+ createCubeTable(factTable, storageTableDescs);
+ // do a get to update cache
+ getCubeFact(factName);
+
+ }
public <T extends Equals & HashCode & ToString> void createEntity(T entity) throws LensException {
if (entity instanceof XStorage) {
@@ -362,17 +373,6 @@
JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()),
JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables()));
}
- public void createCubeFactTable(String cubeName, String factName, List<FieldSchema> columns,
- Map<String, Set<UpdatePeriod>> storageAggregatePeriods, double weight, Map<String, String> properties,
- Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap)
- throws LensException {
- CubeFactTable factTable = new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight,
- properties, storageUpdatePeriodMap);
- createCubeTable(factTable, storageTableDescs);
- // do a get to update cache
- getCubeFact(factName);
-
- }
/**
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
index 76e5f23..e5cf916 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
@@ -141,6 +141,7 @@
Set<Dimension> rewriteDenormctx(CubeQueryContext cubeql,
StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery, boolean replaceFact) throws LensException {
Set<Dimension> refTbls = new HashSet<>();
+ log.info("Doing denorm changes for fact :{}", sc);
if (!tableToRefCols.isEmpty()) {
// pick referenced columns for fact
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 97a9ef0..aaa183b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -393,7 +393,7 @@
pickExpressionsForTable(cdim);
}
}
- log.debug("Picked expressions: {}", pickedExpressions);
+ log.info("Picked expressions: {}", pickedExpressions);
for (Set<PickedExpression> peSet : pickedExpressions.values()) {
for (PickedExpression pe : peSet) {
exprDims.addAll(pe.pickedCtx.exprDims);
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 860db28..62d7386 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -525,8 +525,6 @@
return expected.toString();
}
- private Set<ExprColumn> exprs;
-
private void assertTestFactTimelineClass(CubeMetastoreClient client) throws Exception {
String factName = "testFact";
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index abaae5b..194b380 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -226,8 +226,6 @@
*/
@Override
public void createDimensionTable(LensSessionHandle sessionid, XDimensionTable xDimTable) throws LensException {
- String dimTblName = xDimTable.getTableName();
-
try (SessionContext ignored = new SessionContext(sessionid)){
getClient(sessionid).createCubeDimensionTable(xDimTable);
log.info("Dimension Table created " + xDimTable.getTableName());
diff --git a/tools/conf/server/lens-site.xml b/tools/conf/server/lens-site.xml
index b53ff25..c303198 100644
--- a/tools/conf/server/lens-site.xml
+++ b/tools/conf/server/lens-site.xml
@@ -36,4 +36,10 @@
<name>datanucleus.schema.autoCreateTables</name>
<value>true</value>
</property>
+
+<property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+</property>
+
</configuration>
