Merge branch 'develop' of https://github.com/InMobi/grill into develop
diff --git a/grill-server-api/src/main/java/com/inmobi/grill/server/api/GrillConfConstants.java b/grill-server-api/src/main/java/com/inmobi/grill/server/api/GrillConfConstants.java
index 617d9bb..96b0c8e 100644
--- a/grill-server-api/src/main/java/com/inmobi/grill/server/api/GrillConfConstants.java
+++ b/grill-server-api/src/main/java/com/inmobi/grill/server/api/GrillConfConstants.java
@@ -181,7 +181,7 @@
public static final String DEFAULT_STATISTICS_DATABASE = "grillstats";
- public static final String GRILL_STATS_ROLLUP_SCAN_RATE = "grill.log.rollover.interval";
+ public static final String GRILL_STATS_ROLLUP_SCAN_RATE = "grill.statistics.log.rollover.interval";
public static final long DEFAULT_STATS_ROLLUP_SCAN_RATE = 3600000;
diff --git a/grill-server-api/src/main/java/com/inmobi/grill/server/api/query/QueryContext.java b/grill-server-api/src/main/java/com/inmobi/grill/server/api/query/QueryContext.java
index 15b18dc..89dfc82 100644
--- a/grill-server-api/src/main/java/com/inmobi/grill/server/api/query/QueryContext.java
+++ b/grill-server-api/src/main/java/com/inmobi/grill/server/api/query/QueryContext.java
@@ -67,6 +67,7 @@
@Getter @Setter private String driverOpHandle;
@Getter final DriverQueryStatus driverStatus;
transient @Getter @Setter private QueryOutputFormatter queryOutputFormatter;
+ @Getter @Setter private boolean finishedQueryPersisted = false;
public QueryContext(String query, String user, Configuration conf) {
this(query, user, new GrillConf(), conf, query, null);
diff --git a/grill-server/src/main/java/com/inmobi/grill/server/query/QueryEndNotifier.java b/grill-server/src/main/java/com/inmobi/grill/server/query/QueryEndNotifier.java
index 91f6e1b..090d5b1 100644
--- a/grill-server/src/main/java/com/inmobi/grill/server/query/QueryEndNotifier.java
+++ b/grill-server/src/main/java/com/inmobi/grill/server/query/QueryEndNotifier.java
@@ -64,7 +64,15 @@
@Override
public void process(QueryEnded event) {
+ if (event.getCurrentValue() == QueryStatus.Status.CLOSED) {
+ return;
+ }
QueryContext queryContext = queryService.getQueryContext(event.getQueryHandle());
+ if (queryContext == null) {
+ LOG.warn("Could not find the context for " + event.getQueryHandle() + " for event:"
+ + event.getCurrentValue() + ". No email generated");
+ return;
+ }
boolean whetherMailNotify = Boolean.parseBoolean(queryContext.getConf().get(GrillConfConstants.GRILL_WHETHER_MAIL_NOTIFY,
GrillConfConstants.GRILL_WHETHER_MAIL_NOTIFY_DEFAULT));
diff --git a/grill-server/src/main/java/com/inmobi/grill/server/query/QueryExecutionServiceImpl.java b/grill-server/src/main/java/com/inmobi/grill/server/query/QueryExecutionServiceImpl.java
index ba79b5d..039790a 100644
--- a/grill-server/src/main/java/com/inmobi/grill/server/query/QueryExecutionServiceImpl.java
+++ b/grill-server/src/main/java/com/inmobi/grill/server/query/QueryExecutionServiceImpl.java
@@ -485,32 +485,21 @@
return;
}
try {
- FinishedGrillQuery finishedQuery = new FinishedGrillQuery(finished.getCtx());
- if (finished.ctx.getStatus().getStatus()
- == Status.SUCCESSFUL) {
- if (finished.ctx.getStatus().isResultSetAvailable()) {
- GrillResultSet set = getResultset(finished.getCtx().getQueryHandle());
- if(set != null &&PersistentResultSet.class.isAssignableFrom(set.getClass())) {
- GrillResultSetMetadata metadata = set.getMetadata();
- String outputPath = ((PersistentResultSet) set).getOutputPath();
- int rows = set.size();
- finishedQuery.setMetadataClass(metadata.getClass().getName());
- finishedQuery.setResult(outputPath);
- finishedQuery.setMetadata(mapper.writeValueAsString(metadata));
- finishedQuery.setRows(rows);
- }
+ FinishedGrillQuery finishedQuery = new FinishedGrillQuery(finished.getCtx());
+ if (finished.ctx.getStatus().getStatus()
+ == Status.SUCCESSFUL) {
+ if (finished.ctx.getStatus().isResultSetAvailable()) {
+ GrillResultSet set = getResultset(finished.getCtx().getQueryHandle());
+ if(set != null &&PersistentResultSet.class.isAssignableFrom(set.getClass())) {
+ GrillResultSetMetadata metadata = set.getMetadata();
+ String outputPath = ((PersistentResultSet) set).getOutputPath();
+ int rows = set.size();
+ finishedQuery.setMetadataClass(metadata.getClass().getName());
+ finishedQuery.setResult(outputPath);
+ finishedQuery.setMetadata(mapper.writeValueAsString(metadata));
+ finishedQuery.setRows(rows);
}
}
-
- // session is not required to close the query
- //acquire(finished.getCtx().getGrillSessionIdentifier());
- try {
- if (finished.getCtx().getSelectedDriver() != null) {
- finished.getCtx().getSelectedDriver().closeQuery(
- finished.getCtx().getQueryHandle());
- }
- } catch (GrillException e) {
- LOG.warn("Exception while closing query with selected driver.", e);
}
try {
grillServerDao.insertFinishedQuery(finishedQuery);
@@ -519,8 +508,20 @@
finishedQueries.add(finished);
continue;
}
- allQueries.remove(finished.getCtx().getQueryHandle());
- resultSets.remove(finished.getCtx().getQueryHandle());
+
+ synchronized (finished.ctx) {
+ finished.ctx.setFinishedQueryPersisted(true);
+ try {
+ if (finished.getCtx().getSelectedDriver() != null) {
+ finished.getCtx().getSelectedDriver().closeQuery(
+ finished.getCtx().getQueryHandle());
+ }
+ } catch (Exception e) {
+ LOG.warn("Exception while closing query with selected driver.", e);
+ }
+ allQueries.remove(finished.getCtx().getQueryHandle());
+ resultSets.remove(finished.getCtx().getQueryHandle());
+ }
fireStatusChangeEvent(finished.getCtx(),
new QueryStatus(1f, Status.CLOSED, "Query purged", false, null, null),
finished.getCtx().getStatus());
@@ -693,45 +694,53 @@
getEventService().notifyEvent(new QueryAccepted(System.currentTimeMillis(), null, query, null));
}
-
+ private GrillResultSet getResultsetFromDAO(QueryHandle queryHandle) throws GrillException {
+ FinishedGrillQuery query = grillServerDao.getQuery(queryHandle.toString());
+ if(query != null) {
+ if(query.getResult() == null) {
+ throw new NotFoundException("InMemory Query result purged " + queryHandle);
+ }
+ try {
+ Class<GrillResultSetMetadata> mdKlass =
+ (Class<GrillResultSetMetadata>) Class.forName(query.getMetadataClass());
+ return new GrillPersistentResult(
+ mapper.readValue(query.getMetadata(), mdKlass),
+ query.getResult(), query.getRows());
+ } catch (Exception e) {
+ throw new GrillException(e);
+ }
+ }
+ throw new NotFoundException("Query not found: " + queryHandle);
+ }
private GrillResultSet getResultset(QueryHandle queryHandle)
throws GrillException {
- if (!allQueries.containsKey(queryHandle)) {
- FinishedGrillQuery query = grillServerDao.getQuery(queryHandle.toString());
- if(query != null) {
- if(query.getResult() == null) {
- throw new NotFoundException("InMemory Query result purged " + queryHandle);
+ QueryContext ctx = allQueries.get(queryHandle);
+ if (ctx == null) {
+ return getResultsetFromDAO(queryHandle);
+ } else {
+ synchronized (ctx) {
+ if (ctx.isFinishedQueryPersisted()) {
+ return getResultsetFromDAO(queryHandle);
}
- try {
- Class<GrillResultSetMetadata> mdKlass =
- (Class<GrillResultSetMetadata>) Class.forName(query.getMetadataClass());
- return new GrillPersistentResult(
- mapper.readValue(query.getMetadata(), mdKlass),
- query.getResult(), query.getRows());
- } catch (Exception e) {
- throw new GrillException(e);
+ GrillResultSet resultSet = resultSets.get(queryHandle);
+ if (resultSet == null) {
+ if (ctx.isPersistent() && ctx.getQueryOutputFormatter() != null) {
+ resultSets.put(queryHandle, new GrillPersistentResult(
+ ctx.getQueryOutputFormatter().getMetadata(),
+ ctx.getQueryOutputFormatter().getFinalOutputPath().toString(),
+ ctx.getQueryOutputFormatter().getNumRows()));
+ } else if (allQueries.get(queryHandle).isResultAvailableInDriver()) {
+ resultSet = allQueries.get(queryHandle).getSelectedDriver().
+ fetchResultSet(allQueries.get(queryHandle));
+ resultSets.put(queryHandle, resultSet);
+ } else {
+ throw new NotFoundException("Result set not available for query:" + queryHandle);
+ }
}
}
- throw new NotFoundException("Query not found: " + queryHandle);
+ return resultSets.get(queryHandle);
}
- GrillResultSet resultSet = resultSets.get(queryHandle);
- if (resultSet == null) {
- QueryContext ctx = allQueries.get(queryHandle);
- if (ctx.isPersistent() && ctx.getQueryOutputFormatter() != null) {
- resultSets.put(queryHandle, new GrillPersistentResult(
- ctx.getQueryOutputFormatter().getMetadata(),
- ctx.getQueryOutputFormatter().getFinalOutputPath().toString(),
- ctx.getQueryOutputFormatter().getNumRows()));
- } else if (allQueries.get(queryHandle).isResultAvailableInDriver()) {
- resultSet = allQueries.get(queryHandle).getSelectedDriver().
- fetchResultSet(allQueries.get(queryHandle));
- resultSets.put(queryHandle, resultSet);
- } else {
- throw new NotFoundException("Result set not available for query:" + queryHandle);
- }
- }
- return resultSets.get(queryHandle);
}
GrillResultSet getDriverResultset(QueryHandle queryHandle)
@@ -990,7 +999,7 @@
e.printStackTrace();
}
}
- QueryCompletionListener listener = new QueryCompletionListenerImpl();
+ QueryCompletionListener listener = new QueryCompletionListenerImpl(handle);
getQueryContext(sessionHandle, handle).getSelectedDriver().
registerForCompletionNotification(handle, timeoutMillis, listener);
try {
@@ -1009,13 +1018,15 @@
class QueryCompletionListenerImpl implements QueryCompletionListener {
boolean succeeded = false;
- QueryCompletionListenerImpl() {
+ QueryHandle handle;
+ QueryCompletionListenerImpl(QueryHandle handle) {
+ this.handle = handle;
}
@Override
public void onCompletion(QueryHandle handle) {
synchronized (this) {
succeeded = true;
- LOG.info("Query with time out succeeded");
+ LOG.info("Query " + handle + " with time out succeeded");
this.notify();
}
}
@@ -1024,7 +1035,7 @@
public void onError(QueryHandle handle, String error) {
synchronized (this) {
succeeded = false;
- LOG.info("Query with time out failed");
+ LOG.info("Query " + handle + " with time out failed");
this.notify();
}
}
diff --git a/grill-server/src/main/java/com/inmobi/grill/server/query/QueryExecutionStatisticsGenerator.java b/grill-server/src/main/java/com/inmobi/grill/server/query/QueryExecutionStatisticsGenerator.java
index 9e4c9a4..7ad7b67 100644
--- a/grill-server/src/main/java/com/inmobi/grill/server/query/QueryExecutionStatisticsGenerator.java
+++ b/grill-server/src/main/java/com/inmobi/grill/server/query/QueryExecutionStatisticsGenerator.java
@@ -62,6 +62,7 @@
return;
}
event.setEndTime(ctx.getEndTime());
+ event.setStartTime(ctx.getLaunchTime());
event.setStatus(ctx.getStatus());
event.setCause(ended.getCause() != null ? ended.getCause() : "");
event.setResult(ctx.getResultSetPath());
diff --git a/grill-server/src/main/resources/grillserver-default.xml b/grill-server/src/main/resources/grillserver-default.xml
index 4885cc6..309570e 100644
--- a/grill-server/src/main/resources/grillserver-default.xml
+++ b/grill-server/src/main/resources/grillserver-default.xml
@@ -183,7 +183,7 @@
</property>
<property>
- <name>grill.log.rollover.interval</name>
+ <name>grill.statistics.log.rollover.interval</name>
<value>3600000</value>
<description>Default rate which log statistics store scans for rollups in milliseconds.</description>
</property>
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index 7e623d6..a9a000c 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -32,64 +32,74 @@
*--+--+---+--+
|4|grill.event.service.thread.pool.size| |The size of thread pool for notifying events. The no value is specified, it uses the available processors as the number.|
*--+--+---+--+
-|5|grill.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
+|5|grill.mail.from.address|blah@company.com|The from field in the notifier mail to the submitter.|
*--+--+---+--+
-|6|grill.max.finished.queries|100|Maximum number of finished queries which grill server will keep in memory before purging.|
+|6|grill.mail.host|mail-host.company.com|SMTP Host for sending mail|
*--+--+---+--+
-|7|grill.metastore.service.impl|com.inmobi.grill.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service|
+|7|grill.mail.port|25|SMTP Port|
*--+--+---+--+
-|8|grill.metrics.ganglia.host| |The ganglia host name|
+|8|grill.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.|
*--+--+---+--+
-|9|grill.metrics.ganglia.port| |The ganglia port|
+|9|grill.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.|
*--+--+---+--+
-|10|grill.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds|
+|10|grill.max.finished.queries|100|Maximum number of finished queries which grill server will keep in memory before purging.|
*--+--+---+--+
-|11|grill.query.service.impl|com.inmobi.grill.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
+|11|grill.metastore.service.impl|com.inmobi.grill.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service|
*--+--+---+--+
-|12|grill.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class com.inmobi.grill.server.query.QueryExecutionServiceImpl.QueryStatusLogger |
+|12|grill.metrics.ganglia.host| |The ganglia host name|
*--+--+---+--+
-|13|grill.quota.service.impl|com.inmobi.grill.server.quota.QuotaServiceImpl|Implementation class for quota service|
+|13|grill.metrics.ganglia.port| |The ganglia port|
*--+--+---+--+
-|14|grill.scheduler.service.impl|com.inmobi.grill.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
+|14|grill.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds|
*--+--+---+--+
-|15|grill.server.base.url|http://localhost:9999/grillapi/|The base url for the grill server|
+|15|grill.query.service.impl|com.inmobi.grill.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
*--+--+---+--+
-|16|grill.server.db.driver.name|org.hsqldb.jdbcDriver|Database driver for database where Finished queries have to be stored.|
+|16|grill.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class com.inmobi.grill.server.query.QueryExecutionServiceImpl.QueryStatusLogger |
*--+--+---+--+
-|17|grill.server.db.jdbc.pass| |JDBC Password for Finished queries table|
+|17|grill.quota.service.impl|com.inmobi.grill.server.quota.QuotaServiceImpl|Implementation class for quota service|
*--+--+---+--+
-|18|grill.server.db.jdbc.url|jdbc:hsqldb:/tmp/grillserver/queries.db|JDBC URL where the database for storing finished queries is located.|
+|18|grill.scheduler.service.impl|com.inmobi.grill.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
*--+--+---+--+
-|19|grill.server.db.jdbc.user|SA|JDBC User for Finished queries table|
+|19|grill.server.base.url|http://localhost:9999/grillapi/|The base url for the grill server|
*--+--+---+--+
-|20|grill.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests. |
+|20|grill.server.db.driver.name|org.hsqldb.jdbcDriver|Database driver for database where Finished queries have to be stored.|
*--+--+---+--+
-|21|grill.server.persist.location|file:///tmp/grillserver|The directory in which grill server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location |
+|21|grill.server.db.jdbc.pass| |JDBC Password for Finished queries table|
*--+--+---+--+
-|22|grill.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
+|22|grill.server.db.jdbc.url|jdbc:hsqldb:/tmp/grillserver/queries.db|JDBC URL where the database for storing finished queries is located.|
*--+--+---+--+
-|23|grill.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed. |
+|23|grill.server.db.jdbc.user|SA|JDBC User for Finished queries table|
*--+--+---+--+
-|24|grill.server.session.timeout.seconds|86400|Grill session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
+|24|grill.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests. |
*--+--+---+--+
-|25|grill.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving grill server state.|
+|25|grill.server.persist.location|file:///tmp/grillserver|The directory in which grill server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location |
*--+--+---+--+
-|26|grill.server.ui.base.uri|http://localhost:19999/|The base url for the Grill UI Server|
+|26|grill.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
*--+--+---+--+
-|27|grill.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
+|27|grill.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed. |
*--+--+---+--+
-|28|grill.server.ui.static.dir|webapp/grill-server/static|The base directory to server UI static files from|
+|28|grill.server.session.timeout.seconds|86400|Grill session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
*--+--+---+--+
-|29|grill.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when grill-server starts up|
+|29|grill.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving grill server state.|
*--+--+---+--+
-|30|grill.session.service.impl|com.inmobi.grill.server.session.HiveSessionService|Implementation class for session service|
+|30|grill.server.ui.base.uri|http://localhost:19999/|The base url for the Grill UI Server|
*--+--+---+--+
-|31|grill.statistics.db|grillstats|Database to which statistics tables are created and partitions are added.|
+|31|grill.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
*--+--+---+--+
-|32|grill.statistics.store.class|com.inmobi.grill.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Grill Statistics.|
+|32|grill.server.ui.static.dir|webapp/grill-server/static|The base directory to server UI static files from|
*--+--+---+--+
-|33|grill.statistics.warehouse.dir|file:///tmp/grill/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
+|33|grill.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when grill-server starts up|
*--+--+---+--+
-|34|hive.server2.log.redirection.enabled|false|Disable the log direction on the grill server end to decrease number of file handles associated to grill server.|
+|34|grill.session.service.impl|com.inmobi.grill.server.session.HiveSessionService|Implementation class for session service|
+*--+--+---+--+
+|35|grill.statistics.db|grillstats|Database to which statistics tables are created and partitions are added.|
+*--+--+---+--+
+|36|grill.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
+*--+--+---+--+
+|37|grill.statistics.store.class|com.inmobi.grill.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Grill Statistics.|
+*--+--+---+--+
+|38|grill.statistics.warehouse.dir|file:///tmp/grill/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
+*--+--+---+--+
+|39|hive.server2.log.redirection.enabled|false|Disable the log direction on the grill server end to decrease number of file handles associated to grill server.|
*--+--+---+--+
The configuration parameters and their default values
diff --git a/src/site/apt/admin/session-config.apt b/src/site/apt/admin/session-config.apt
index 44fc98a..d258ba3 100644
--- a/src/site/apt/admin/session-config.apt
+++ b/src/site/apt/admin/session-config.apt
@@ -66,7 +66,7 @@
*--+--+---+--+
|21|grill.result.output.serde|com.inmobi.grill.lib.query.CSVSerde|The default serde class name that should be used by com.inmobi.grill.lib.query.FileSerdeFormatter for formatting the output |
*--+--+---+--+
-|22|grill.result.parent.dir|/tmp/grillreports|The directory for storing persisted result of query.|
+|22|grill.result.parent.dir|/tmp/grillreports|The directory for storing persisted result of query. This directory should exist and should have writable permissions by grill server|
*--+--+---+--+
|23|grill.whether.mail.notify|false|When a query ends, whether to notify the submitter by mail or not.|
*--+--+---+--+
diff --git a/tools/conf/server/log4j.properties b/tools/conf/server/log4j.properties
index fbe28a0..d178c18 100644
--- a/tools/conf/server/log4j.properties
+++ b/tools/conf/server/log4j.properties
@@ -61,6 +61,6 @@
#Add query statistics logger with hourly rollup
log4j.appender.QueryExecutionStatistics=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.QueryExecutionStatistics.DatePattern='.'yyyy-MM-dd-HH-mm
+log4j.appender.QueryExecutionStatistics.DatePattern='.'yyyy-MM-dd-HH
log4j.appender.QueryExecutionStatistics.File=${grill.log.dir}/query-stats.log
log4j.appender.QueryExecutionStatistics.layout=com.inmobi.grill.server.stats.store.log.StatisticsLogLayout
