| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * <p> |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * <p> |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.atlas.services; |
| |
| import com.google.common.annotations.VisibleForTesting; |
| import com.google.inject.Singleton; |
| import org.apache.atlas.ApplicationProperties; |
| import org.apache.atlas.AtlasException; |
| import org.apache.atlas.exception.AtlasBaseException; |
| import org.apache.atlas.model.metrics.AtlasMetrics; |
| import org.apache.atlas.repository.graph.AtlasGraphProvider; |
| import org.apache.atlas.repository.graphdb.AtlasGraph; |
| import org.apache.atlas.util.AtlasGremlinQueryProvider; |
| import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery; |
| import org.apache.commons.configuration.Configuration; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| import javax.inject.Inject; |
| import java.util.List; |
| import java.util.Map; |
| |
| @Singleton |
| public class MetricsService { |
| private static final Logger LOG = LoggerFactory.getLogger(MetricsService.class); |
| |
| // Query Category constants |
| public static final String TYPE = "type"; |
| public static final String ENTITY = "entity"; |
| public static final String TAG = "tag"; |
| public static final String GENERAL = "general"; |
| |
| // Query names |
| protected static final String METRIC_TYPE_COUNT = TYPE + "Count"; |
| protected static final String METRIC_TYPE_UNUSED_COUNT = TYPE + "UnusedCount"; |
| protected static final String METRIC_TYPE_ENTITIES = TYPE + "Entities"; |
| |
| protected static final String METRIC_ENTITY_COUNT = ENTITY + "Count"; |
| protected static final String METRIC_ENTITY_DELETED = ENTITY + "Deleted"; |
| protected static final String METRIC_TAGGED_ENTITIES = ENTITY + "Tagged"; |
| protected static final String METRIC_TAGS_PER_ENTITY = ENTITY + "Tags"; |
| |
| protected static final String METRIC_TAG_COUNT = TAG + "Count"; |
| protected static final String METRIC_ENTITIES_PER_TAG = TAG + "Entities"; |
| |
| public static final String METRIC_QUERY_PREFIX = "atlas.metric.query."; |
| public static final String METRIC_QUERY_CACHE_TTL = "atlas.metric.query.cache.ttlInSecs"; |
| public static final int DEFAULT_CACHE_TTL_IN_SECS = 900; |
| |
| public static final String METRIC_COLLECTION_TIME = "collectionTime"; |
| |
| private static Configuration configuration = null; |
| private static AtlasGremlinQueryProvider gremlinQueryProvider = null; |
| |
| private final AtlasGraph atlasGraph; |
| private final int cacheTTLInSecs; |
| |
| private AtlasMetrics cachedMetrics = null; |
| private long cacheExpirationTime = 0; |
| |
| |
| @Inject |
| public MetricsService() throws AtlasException { |
| this(ApplicationProperties.get(), AtlasGraphProvider.getGraphInstance()); |
| } |
| |
| @VisibleForTesting |
| MetricsService(Configuration configuration, AtlasGraph graph) { |
| MetricsService.configuration = configuration; |
| |
| atlasGraph = graph; |
| cacheTTLInSecs = configuration != null ? configuration.getInt(METRIC_QUERY_CACHE_TTL, DEFAULT_CACHE_TTL_IN_SECS) |
| : DEFAULT_CACHE_TTL_IN_SECS; |
| gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE; |
| } |
| |
| @SuppressWarnings("unchecked") |
| public AtlasMetrics getMetrics(boolean ignoreCache) { |
| if (ignoreCache || !isCacheValid()) { |
| AtlasMetrics metrics = new AtlasMetrics(); |
| |
| for (MetricQuery metricQuery : MetricQuery.values()) { |
| try { |
| if (LOG.isDebugEnabled()) { |
| LOG.debug("Executing query: {}", metricQuery); |
| } |
| executeGremlinQuery(metrics, metricQuery.group, metricQuery.name, metricQuery.query); |
| } catch (AtlasBaseException e) { |
| if (LOG.isDebugEnabled()) { |
| LOG.debug("Gremlin execution failed for metric {}", metricQuery, e); |
| } else { |
| LOG.warn("Gremlin execution failed for metric {}", metricQuery); |
| } |
| } |
| } |
| |
| long collectionTime = System.currentTimeMillis(); |
| |
| metrics.addData(GENERAL, METRIC_COLLECTION_TIME, collectionTime); |
| |
| this.cachedMetrics = metrics; |
| this.cacheExpirationTime = (collectionTime + cacheTTLInSecs * 1000); |
| } |
| |
| return cachedMetrics; |
| } |
| |
| private void executeGremlinQuery(AtlasMetrics metrics, String type, String name, String query) throws AtlasBaseException { |
| Object result = atlasGraph.executeGremlinScript(query, false); |
| |
| if (result instanceof Number) { |
| metrics.addData(type, name, ((Number) result).intValue()); |
| } else if (result instanceof List) { |
| for (Map<String, Number> resultMap : (List<Map<String, Number>>) result) { |
| for (Map.Entry<String, Number> entry : resultMap.entrySet()) { |
| metrics.addData(type, entry.getKey(), entry.getValue().intValue()); |
| } |
| } |
| } else { |
| String returnClassName = result != null ? result.getClass().getSimpleName() : "null"; |
| |
| LOG.warn("Unhandled return type {} for {}. Ignoring", returnClassName, query); |
| } |
| } |
| |
| private boolean isCacheValid() { |
| boolean valid = cachedMetrics != null && System.currentTimeMillis() < cacheExpirationTime; |
| |
| if (LOG.isDebugEnabled()) { |
| LOG.debug("cachedMetrics: {}", cachedMetrics != null); |
| LOG.debug("cacheExpirationTime: {}", cacheExpirationTime); |
| LOG.debug("valid: {}", valid); |
| } |
| |
| return valid; |
| } |
| |
| private static String getQuery(String type, String name, String defaultQuery) { |
| String ret = configuration != null ? configuration.getString(METRIC_QUERY_PREFIX + type + "." + name, defaultQuery) |
| : defaultQuery; |
| |
| if (LOG.isDebugEnabled()) { |
| LOG.debug("query for {}.{}: {}", type, name, ret); |
| } |
| |
| return ret; |
| } |
| |
| /** |
| * MetricQuery enum has the capability of reading the queries from the externalized config. |
| * |
| * The default behavior is to read from the properties and override the statically type query if the configured |
| * query is not blank/empty. |
| */ |
| private enum MetricQuery { |
| TYPE_COUNT(GENERAL, METRIC_TYPE_COUNT, AtlasGremlinQuery.TYPE_COUNT_METRIC), |
| UNUSED_TYPE_COUNT(GENERAL, METRIC_TYPE_UNUSED_COUNT, AtlasGremlinQuery.TYPE_UNUSED_COUNT_METRIC), |
| ENTITY_COUNT(GENERAL, METRIC_ENTITY_COUNT, AtlasGremlinQuery.ENTITY_COUNT_METRIC), |
| TAGS_COUNT(GENERAL, METRIC_TAG_COUNT, AtlasGremlinQuery.TAG_COUNT_METRIC), |
| DELETED_ENTITY_COUNT(GENERAL, METRIC_ENTITY_DELETED, AtlasGremlinQuery.ENTITY_DELETED_METRIC), |
| |
| ENTITIES_PER_TYPE(ENTITY, METRIC_TYPE_ENTITIES, AtlasGremlinQuery.ENTITIES_PER_TYPE_METRIC), |
| TAGGED_ENTITIES(ENTITY, METRIC_TAGGED_ENTITIES, AtlasGremlinQuery.TAGGED_ENTITIES_METRIC), |
| |
| ENTITIES_WITH_SPECIFIC_TAG(TAG, METRIC_ENTITIES_PER_TAG, AtlasGremlinQuery.ENTITIES_FOR_TAG_METRIC), |
| ; |
| |
| private final String group; |
| private final String name; |
| private final String query; |
| |
| MetricQuery(String group, String name, AtlasGremlinQuery gremlinQuery) { |
| this.group = group; |
| this.name = name; |
| this.query = MetricsService.getQuery(group, name, gremlinQueryProvider.getQuery(gremlinQuery)); |
| } |
| |
| @Override |
| public String toString() { |
| return "MetricQuery{" + "group='" + group + '\'' + |
| ", name='" + name + '\'' + |
| ", query='" + query + '\'' + |
| '}'; |
| } |
| } |
| } |