blob: 76c8cad71305157414da8254cdcaf63e0e1e578d [file] [log] [blame]
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lens.client;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.ws.rs.core.Response;
import org.apache.lens.api.APIResult;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.metastore.*;
import org.apache.lens.api.query.*;
import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.api.util.PathValidator;
import org.apache.lens.client.exceptions.LensAPIException;
import org.apache.lens.client.exceptions.LensBriefErrorException;
import org.apache.lens.client.exceptions.LensClientIOException;
import org.apache.lens.client.model.BriefError;
import org.apache.lens.client.model.IdBriefErrorTemplate;
import org.apache.lens.client.model.IdBriefErrorTemplateKey;
import org.apache.lens.client.resultset.CsvResultSet;
import org.apache.lens.client.resultset.ResultSet;
import org.apache.lens.client.resultset.ZippedCsvResultSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LensClient implements AutoCloseable {
public static final String CLILOGGER = "cliLogger";
private static final String DEFAULT_PASSWORD = "";
@Getter
private final LensClientConfig conf;
@Getter
private final LensMetadataClient mc;
private String password;
@Getter
private LensConnection connection;
private final HashMap<QueryHandle, LensStatement> statementMap =
Maps.newHashMap();
@Getter
private final LensStatement statement;
@Getter
private PathValidator pathValidator;
public static final String QUERY_RESULT_SPLIT_INTO_MULTIPLE = "lens.query.result.split.multiple";
public static final String QUERY_OUTPUT_WRITE_HEADER_ENABLED = "lens.query.output.write.header";
public static final String QUERY_OUTPUT_ENCODING = "lens.query.output.charset.encoding";
public static final char DEFAULT_RESULTSET_DELIMITER = ',';
public static Logger getCliLogger() {
return LoggerFactory.getLogger(CLILOGGER);
}
public LensClient() {
this(new LensClientConfig());
}
public LensClient(LensClientConfig conf) {
this(conf, conf.getUser(), DEFAULT_PASSWORD);
}
public LensClient(String username, String password) {
this(new LensClientConfig(), username, password);
}
public LensClient(LensClientConfig conf, String username, String password) {
this.conf = conf;
conf.setUser(username);
this.password = password;
connectToLensServer();
mc = new LensMetadataClient(connection);
statement = new LensStatement(connection);
}
public LensClient(Credentials cred) {
this(cred.getUsername(), cred.getPassword());
}
public LensMetadataClient getMetadataClient() {
return mc;
}
@Deprecated
public QueryHandle executeQueryAsynch(String sql, String queryName) throws LensAPIException {
return executeQueryAsynch(sql, queryName, new LensConf());
}
public QueryHandle executeQueryAsynch(String sql, String queryName, LensConf conf) throws LensAPIException {
log.debug("Executing query {}", sql);
QueryHandle handle = statement.executeQuery(sql, false, queryName, conf);
statementMap.put(handle, statement);
return handle;
}
/**
* Execute query with timeout option.
* If the query does not finish within the timeout time, server returns the query handle which can be used to
* track further progress.
*
* @param sql : query/command to be executed
* @param queryName : optional query name
* @param timeOutMillis : timeout milliseconds for the query execution.
* @return
* @throws LensAPIException
*/
@Deprecated
public QueryHandleWithResultSet executeQueryWithTimeout(String sql, String queryName, long timeOutMillis)
throws LensAPIException {
return executeQueryWithTimeout(sql, queryName, timeOutMillis, new LensConf());
}
/**
* Execute query with timeout option.
* If the query does not finish within the timeout time, server returns the query handle which can be used to
* track further progress.
*
* @param sql : query/command to be executed
* @param queryName : optional query name
* @param timeOutMillis : timeout milliseconds for the query execution.
* @param conf config to be used for the query
* @return
* @throws LensAPIException
*/
public QueryHandleWithResultSet executeQueryWithTimeout(String sql, String queryName,
long timeOutMillis, LensConf conf) throws LensAPIException {
log.info("Executing query {} with timeout of {} milliseconds", sql, timeOutMillis);
QueryHandleWithResultSet result = statement.executeQuery(sql, queryName, timeOutMillis, conf);
statementMap.put(result.getQueryHandle(), statement);
if (result.getStatus().failed()) {
IdBriefErrorTemplate errorResult = new IdBriefErrorTemplate(IdBriefErrorTemplateKey.QUERY_ID,
result.getQueryHandle().getHandleIdString(), new BriefError(result.getStatus()
.getErrorCode(), result.getStatus().getErrorMessage()));
throw new LensBriefErrorException(errorResult);
}
return result;
}
public Date getLatestDateOfCube(String cubeName, String timePartition) {
return mc.getLatestDateOfCube(cubeName, timePartition);
}
public List<String> getPartitionTimelines(String factName, String storageName, String updatePeriod,
String timeDimension) {
return mc.getPartitionTimelines(factName, storageName, updatePeriod, timeDimension);
}
public static class LensClientResultSetWithStats {
private final LensClientResultSet resultSet;
private final LensQuery query;
public LensClientResultSetWithStats(LensClientResultSet resultSet,
LensQuery query) {
this.resultSet = resultSet;
this.query = query;
}
public LensClientResultSet getResultSet() {
return resultSet;
}
public LensQuery getQuery() {
return query;
}
}
public LensClientResultSetWithStats getResults(String sql, String queryName) throws LensAPIException {
log.debug("Executing query {}", sql);
statement.executeQuery(sql, true, queryName);
return getResultsFromStatement(statement);
}
private LensClientResultSetWithStats getResultsFromStatement(LensStatement statement) {
QueryStatus.Status status = statement.getStatus().getStatus();
if (status != QueryStatus.Status.SUCCESSFUL) {
IdBriefErrorTemplate errorResult = new IdBriefErrorTemplate(IdBriefErrorTemplateKey.QUERY_ID,
statement.getQueryHandleString(), new BriefError(statement.getErrorCode(), statement.getErrorMessage()));
throw new LensBriefErrorException(errorResult);
}
LensClientResultSet result = null;
if (statement.getStatus().isResultSetAvailable()) {
result = new LensClientResultSet(statement.getResultSetMetaData(), statement.getResultSet());
}
return new LensClientResultSetWithStats(result, statement.getQuery());
}
private LensClientResultSetWithStats getResultsFromHandle(QueryHandle q, boolean async) {
if (!async) {
statement.waitForQueryToComplete(q);
}
LensQuery query = statement.getQuery(q);
if (query.getStatus().getStatus()
== QueryStatus.Status.FAILED) {
throw new IllegalStateException(query.getStatus().getErrorMessage());
}
LensClientResultSet result = null;
if (query.getStatus().isResultSetAvailable()) {
result = new LensClientResultSet(statement.getResultSetMetaData(query), statement.getResultSet(query));
}
return new LensClientResultSetWithStats(result, query);
}
public LensClientResultSetWithStats getAsyncResults(QueryHandle q) {
return getResultsFromHandle(q, true);
}
public LensClientResultSetWithStats getSyncResults(QueryHandle q) {
return getResultsFromHandle(q, false);
}
public Response getHttpResults() {
return statement.getHttpResultSet();
}
public Response getHttpResults(QueryHandle q) {
return statement.getHttpResultSet(statement.getQuery(q));
}
/**
* Gets the ResultSet for the query represented by queryHandle.
*
* @param queryHandle : query hanlde
* @return
* @throws LensClientIOException
*/
public ResultSet getHttpResultSet(QueryHandle queryHandle) throws LensClientIOException {
Map<String, String> paramsMap = this.connection.getConnectionParamsAsMap();
String isSplitFileEnabled = paramsMap.get(QUERY_RESULT_SPLIT_INTO_MULTIPLE);
String isHeaderEnabled = paramsMap.get(QUERY_OUTPUT_WRITE_HEADER_ENABLED);
String encoding = paramsMap.get(QUERY_OUTPUT_ENCODING);
return getHttpResultSet(queryHandle, Charset.forName(encoding), Boolean.parseBoolean(isHeaderEnabled),
DEFAULT_RESULTSET_DELIMITER, Boolean.parseBoolean(isSplitFileEnabled));
}
/**
* Gets the ResultSet for the query represented by queryHandle.
*
* @param queryHandle : query handle.
* @param encoding : resultset encoding.
* @param isHeaderPresent : whether the resultset has header row included.
* @param delimiter : delimiter used to seperate columns of resultset.
* @param isResultZipped : whether the resultset is zipped.
* @return
* @throws LensClientIOException
*/
public ResultSet getHttpResultSet(QueryHandle queryHandle, Charset encoding, boolean isHeaderPresent, char delimiter,
boolean isResultZipped) throws LensClientIOException {
InputStream resultStream = null;
try {
Response response = statement.getHttpResultSet(statement.getQuery(queryHandle));
resultStream = response.readEntity(InputStream.class);
} catch (Exception e) {
throw new LensClientIOException("Error while getting resultset", e);
}
if (isResultZipped) {
return new ZippedCsvResultSet(resultStream, encoding, isHeaderPresent, delimiter);
} else {
return new CsvResultSet(resultStream, encoding, isHeaderPresent, delimiter);
}
}
public LensStatement getLensStatement(QueryHandle query) {
return this.statementMap.get(query);
}
public QueryStatus getQueryStatus(QueryHandle query) {
return statement.getQuery(query).getStatus();
}
public LensQuery getQueryDetails(QueryHandle handle) {
return statement.getQuery(handle);
}
public QueryStatus getQueryStatus(String q) {
return getQueryStatus(QueryHandle.fromString(q));
}
public LensQuery getQueryDetails(String handle) {
return getQueryDetails(QueryHandle.fromString(handle));
}
public LensAPIResult<QueryPlan> getQueryPlan(String q) throws LensAPIException {
return statement.explainQuery(q, new LensConf());
}
public boolean killQuery(QueryHandle q) {
return statement.kill(statement.getQuery(q));
}
public QueryResult getResults(QueryHandle query) {
QueryStatus status = getLensStatement(query).getStatus();
if (!status.isResultSetAvailable()) {
log.debug("Current status of the query is {}", status);
throw new IllegalStateException("Resultset for the query "
+ query + " is not available, its current status is " + status);
}
return getLensStatement(query).getResultSet();
}
public List<QueryHandle> getQueries(String state, String queryName, String user, String driver, String fromDate,
String toDate) {
return statement.getAllQueries(state, queryName, user, driver, fromDate, toDate);
}
public List<LensQuery> getQueriesWithDetails(String state, String queryName, String user, String driver,
String fromDate, String toDate) {
return statement.getAllQueryDetails(state, queryName, user, driver, fromDate, toDate);
}
private void connectToLensServer() {
log.debug("Connecting to lens server {}", new LensConnectionParams(conf));
connection = new LensConnection(new LensConnectionParams(conf));
connection.open(password);
log.debug("Successfully connected to server {}", connection);
pathValidator = new PathValidator(connection.getLensConnectionParams().getSessionConf());
Preconditions.checkNotNull(pathValidator, "Error in initializing Path Validator.");
}
public List<String> getAllDatabases() {
log.debug("Getting all database");
return mc.getAlldatabases();
}
public List<String> getAllNativeTables() {
log.debug("Getting all native tables");
return mc.getAllNativeTables();
}
public List<String> getAllFactTables() {
log.debug("Getting all fact table");
return mc.getAllFactTables();
}
public List<String> getAllFactTables(String cubeName) {
log.debug("Getting all fact table");
return mc.getAllFactTables(cubeName);
}
public List<String> getAllDimensionTables() {
log.debug("Getting all dimension table");
return mc.getAllDimensionTables();
}
public List<String> getAllDimensionTables(String dimensionName) {
log.debug("Getting all dimension table");
return mc.getAllDimensionTables(dimensionName);
}
public List<String> getAllCubes() {
log.debug("Getting all cubes in database");
return mc.getAllCubes();
}
public List<String> getAllDimensions() {
log.debug("Getting all dimensions in database");
return mc.getAllDimensions();
}
public String getCurrentDatabae() {
log.debug("Getting current database");
return mc.getCurrentDatabase();
}
public boolean setDatabase(String database) {
log.debug("Set the database to {}", database);
APIResult result = mc.setDatabase(database);
return result.getStatus() == APIResult.Status.SUCCEEDED;
}
public APIResult dropDatabase(String database, boolean cascade) {
log.debug("Dropping database {}, cascade: {}", database, cascade);
APIResult result = mc.dropDatabase(database, cascade);
log.debug("Return status of dropping {} result {}", database, result);
return result;
}
public APIResult createDatabase(String database, boolean ignoreIfExists) {
log.debug("Creating database {} ignore {}", database, ignoreIfExists);
APIResult result = mc.createDatabase(database, ignoreIfExists);
log.debug("Create database result {}", result);
return result;
}
public APIResult setConnectionParam(String key, String val) {
return this.connection.setConnectionParams(key, val);
}
public List<String> getConnectionParam() {
return this.connection.getConnectionParams();
}
public List<String> getConnectionParam(String key) {
return this.connection.getConnectionParams(key);
}
public void closeConnection() {
log.debug("Closing lens connection: {}", new LensConnectionParams(conf));
this.connection.close();
}
public APIResult addJarResource(String path) {
return this.connection.addResourceToConnection("jar", path);
}
public APIResult removeJarResource(String path) {
return this.connection.removeResourceFromConnection("jar", path);
}
public APIResult addFileResource(String path) {
return this.connection.addResourceToConnection("file", path);
}
public APIResult removeFileResource(String path) {
return this.connection.removeResourceFromConnection("file", path);
}
public APIResult createFactTable(String factSpec) {
return mc.createFactTable(factSpec);
}
public APIResult createCube(String cubeSpec) {
return mc.createCube(cubeSpec);
}
public APIResult createStorage(String storageSpec) {
return mc.createNewStorage(storageSpec);
}
public APIResult createDimension(String dimSpec) {
return mc.createDimension(dimSpec);
}
public APIResult createDimensionTable(String dimSpec) {
return mc.createDimensionTable(dimSpec);
}
public List<String> getAllStorages() {
return mc.getAllStorages();
}
public APIResult dropDimensionTable(String dim, boolean cascade) {
return mc.dropDimensionTable(dim, cascade);
}
public APIResult dropFactTable(String fact, boolean cascade) {
return mc.dropFactTable(fact, cascade);
}
public APIResult dropCube(String cube) {
return mc.dropCube(cube);
}
public APIResult dropStorage(String storage) {
return mc.dropStorage(storage);
}
public APIResult dropDimension(String dimName) {
return mc.dropDimension(dimName);
}
public APIResult updateFactTable(String factName, String factSpec) {
return mc.updateFactTable(factName, factSpec);
}
public APIResult updateDimensionTable(String dimName, String dimSpec) {
return mc.updateDimensionTable(dimName, dimSpec);
}
public APIResult updateCube(String cubeName, String cubeSpec) {
return mc.updateCube(cubeName, cubeSpec);
}
public APIResult updateStorage(String storageName, String storageSpec) {
return mc.updateStorage(storageName, storageSpec);
}
public APIResult updateDimension(String dimName, String dimSpec) {
return mc.updateDimension(dimName, dimSpec);
}
public XFact getFactTable(String factName) {
return mc.getFactTable(factName);
}
public XDimensionTable getDimensionTable(String dimName) {
return mc.getDimensionTable(dimName);
}
public XNativeTable getNativeTable(String tblName) {
return mc.getNativeTable(tblName);
}
public XCube getCube(String cubeName) {
return mc.getCube(cubeName);
}
public XFlattenedColumns getQueryableFields(String table, boolean flattened) {
return mc.getQueryableFields(table, flattened);
}
public XJoinChains getJoinChains(String table) {
return mc.getJoinChains(table);
}
public XDimension getDimension(String dimName) {
return mc.getDimension(dimName);
}
public XStorage getStorage(String storageName) {
return mc.getStorage(storageName);
}
public List<String> getFactStorages(String fact) {
return mc.getAllStoragesOfFactTable(fact);
}
public List<String> getDimStorages(String dim) {
return mc.getAllStoragesOfDimTable(dim);
}
public APIResult dropAllStoragesOfDim(String table) {
return mc.dropAllStoragesOfDimension(table);
}
public APIResult dropAllStoragesOfFact(String table) {
return mc.dropAllStoragesOfFactTable(table);
}
public APIResult addStorageToFact(String factName, String spec) {
return mc.addStorageToFactTable(factName, spec);
}
public APIResult dropStorageFromFact(String factName, String storage) {
return mc.dropStorageFromFactTable(factName, storage);
}
public XStorageTableElement getStorageFromFact(String fact, String storage) {
return mc.getStorageOfFactTable(fact, storage);
}
public APIResult addStorageToDim(String dim, String storage) {
return mc.addStorageToDimTable(dim, storage);
}
public APIResult dropStorageFromDim(String dim, String storage) {
return mc.dropStoragesOfDimensionTable(dim, storage);
}
public XStorageTableElement getStorageFromDim(String dim, String storage) {
return mc.getStorageOfDimensionTable(dim, storage);
}
public List<XPartition> getAllPartitionsOfFact(String fact, String storage) {
return mc.getPartitionsOfFactTable(fact, storage);
}
public List<XPartition> getAllPartitionsOfFact(String fact, String storage, String list) {
return mc.getPartitionsOfFactTable(fact, storage, list);
}
public List<XPartition> getAllPartitionsOfDim(String dim, String storage) {
return mc.getAllPartitionsOfDimensionTable(dim, storage);
}
public List<XPartition> getAllPartitionsOfDim(String dim, String storage, String list) {
return mc.getAllPartitionsOfDimensionTable(dim, storage);
}
public APIResult dropAllPartitionsOfFact(String fact, String storage) {
return mc.dropPartitionsOfFactTable(fact, storage);
}
public APIResult dropAllPartitionsOfFact(String fact, String storage, String list) {
return mc.dropPartitionsOfFactTable(fact, storage, list);
}
public APIResult dropAllPartitionsOfFact(String fact, String storage, String list, String updatePeriod) {
return mc.dropPartitionsOfFactTable(fact, storage, list, updatePeriod);
}
public APIResult dropAllPartitionsOfDim(String dim, String storage) {
return mc.dropAllPartitionsOfDimensionTable(dim, storage);
}
public APIResult dropAllPartitionsOfDim(String dim, String storage, String list) {
return mc.dropAllPartitionsOfDimensionTable(dim, storage, list);
}
public APIResult addPartitionToFact(String table, String storage, String partSpec) {
return mc.addPartitionToFactTable(table, storage, partSpec);
}
public APIResult addPartitionsToFact(String table, String storage, String partsSpec) {
return mc.addPartitionsToFactTable(table, storage, partsSpec);
}
public APIResult addPartitionToFact(String table, String storage, XPartition xp) {
return mc.addPartitionToFactTable(table, storage, xp);
}
public APIResult addPartitionsToFact(String table, String storage, XPartitionList xpList) {
return mc.addPartitionsToFactTable(table, storage, xpList);
}
public APIResult addPartitionToDim(String table, String storage, String partSpec) {
return mc.addPartitionToDimensionTable(table, storage, partSpec);
}
public APIResult addPartitionToDim(String table, String storage, XPartition xp) {
return mc.addPartitionToDimensionTable(table, storage, xp);
}
public APIResult addPartitionsToDim(String table, String storage, XPartitionList xpList) {
return mc.addPartitionsToDimensionTable(table, storage, xpList);
}
public APIResult addPartitionsToDim(String table, String storage, String partsSpec) {
return mc.addPartitionsToDimensionTable(table, storage, partsSpec);
}
public APIResult updatePartitionOfFact(String table, String storage, String partSpec) {
return mc.updatePartitionOfFactTable(table, storage, partSpec);
}
public APIResult updatePartitionsOfFact(String table, String storage, String partsSpec) {
return mc.updatePartitionsOfFactTable(table, storage, partsSpec);
}
public APIResult updatePartitionOfFact(String table, String storage, XPartition xp) {
return mc.updatePartitionOfFactTable(table, storage, xp);
}
public APIResult updatePartitionsOfFact(String table, String storage, XPartitionList xpList) {
return mc.updatePartitionsOfFactTable(table, storage, xpList);
}
public APIResult updatePartitionOfDim(String table, String storage, String partSpec) {
return mc.updatePartitionOfDimensionTable(table, storage, partSpec);
}
public APIResult updatePartitionOfDim(String table, String storage, XPartition xp) {
return mc.updatePartitionOfDimensionTable(table, storage, xp);
}
public APIResult updatePartitionsOfDim(String table, String storage, XPartitionList xpList) {
return mc.updatePartitionsOfDimensionTable(table, storage, xpList);
}
public APIResult updatePartitionsOfDim(String table, String storage, String partsSpec) {
return mc.updatePartitionsOfDimensionTable(table, storage, partsSpec);
}
@Deprecated
public LensAPIResult<QueryPrepareHandle> prepare(String sql, String queryName) throws LensAPIException {
return prepare(sql, queryName, new LensConf());
}
public LensAPIResult<QueryPrepareHandle> prepare(String sql, String queryName, LensConf conf)
throws LensAPIException {
return statement.prepareQuery(sql, queryName, conf);
}
@Deprecated
public LensAPIResult<QueryPlan> explainAndPrepare(String sql, String queryName) throws LensAPIException {
return explainAndPrepare(sql, queryName, new LensConf());
}
public LensAPIResult<QueryPlan> explainAndPrepare(String sql, String queryName, LensConf conf)
throws LensAPIException {
return statement.explainAndPrepare(sql, queryName);
}
public boolean destroyPrepared(QueryPrepareHandle queryPrepareHandle) {
return statement.destroyPrepared(queryPrepareHandle);
}
public List<QueryPrepareHandle> getPreparedQueries(String userName, String queryName, String fromDate,
String toDate) {
return statement.getAllPreparedQueries(userName, queryName, fromDate, toDate);
}
public LensPreparedQuery getPreparedQuery(QueryPrepareHandle phandle) {
return statement.getPreparedQuery(phandle);
}
public LensClientResultSetWithStats getResultsFromPrepared(QueryPrepareHandle phandle, String queryName) {
QueryHandle qh = statement.executeQuery(phandle, true, queryName);
return getResultsFromHandle(qh, true);
}
@Deprecated
public QueryHandle executePrepared(QueryPrepareHandle phandle, String queryName) {
return executePrepared(phandle, queryName, new LensConf());
}
public QueryHandle executePrepared(QueryPrepareHandle phandle, String queryName, LensConf conf) {
return statement.executeQuery(phandle, false, queryName, conf);
}
public boolean isConnectionOpen() {
return this.connection.isOpen();
}
public List<String> listResources(String type) {
return this.connection.listResourcesFromConnection(type);
}
public Response getLogs(String logFile) {
return this.connection.getLogs(logFile);
}
public XSegmentation getSegmentation(String segName) {
return mc.getSegmentation(segName);
}
public List<String> getAllSegmentations() {
return mc.getAllSegmentations();
}
public List<String> getAllSegmentations(String filter) {
return mc.getAllSegmentations(filter);
}
public APIResult createSegmentation(String segSpec) {
return mc.createSegmentation(segSpec);
}
public APIResult updateSegmentation(String segName, String segSpec) {
return mc.updateSegmentation(segName, segSpec);
}
public APIResult dropSegmentation(String segName) {
return mc.dropSegmentation(segName);
}
@Override
public void close() {
closeConnection();
}
}