Merge commit 'refs/pull/372/head' of github.com:apache/usergrid into two-dot-o-dev
diff --git a/README.md b/README.md
index 59302b6..4f5ba88 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,7 @@
Apache Usergrid
===============
-__WARNING__: This is the __two-dot-o__ branch and work is underway here on a new persistence system for Usergrid. Not everything is working in this branch. If you want stability, you should be working against the master branch or a tag. We refer to the new persistence system as Core Persistence and you can find its modules in the stack/corepersistence directory.
-
+__WARNING__: This is the __two-dot-o__ branch and work is underway here on a new persistence system for Usergrid. We refer to the new persistence system as Core Persistence and you can find its modules in the stack/corepersistence directory.
Overview
--------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationService.java
index 089ca2b..78502fa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationService.java
@@ -36,14 +36,14 @@
* @param applicationScope
* @return
*/
- long sumAllCollections(ApplicationScope applicationScope);
+ long getApplicationSize(ApplicationScope applicationScope);
/**
* get entity size for app
*
* @param applicationScope
* @return
*/
- Map<String,Long> sumEachCollection(ApplicationScope applicationScope);
+ Map<String,Long> getEachCollectionSize(ApplicationScope applicationScope);
/**
* get total entity size for an edge
@@ -52,13 +52,13 @@
* @param edge
* @return
*/
- long sum(final ApplicationScope applicationScope, final SearchEdge edge);
+ long getSize(final ApplicationScope applicationScope, final SearchEdge edge);
/**
- * get sum by collection name
+ * get getSize by collection name
* @param applicationScope
* @param collectionName
* @return
*/
- long getCollectionSum(final ApplicationScope applicationScope, final String collectionName);
+ long getCollectionSize(final ApplicationScope applicationScope, final String collectionName);
}
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java
index 29b48d8..44010db 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/AggregationServiceImpl.java
@@ -22,7 +22,6 @@
import com.codahale.metrics.Timer;
import com.google.common.base.Optional;
import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
@@ -35,8 +34,6 @@
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexLocationStrategy;
import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.impl.SearchEdgeImpl;
-import org.apache.usergrid.utils.IndexUtils;
import rx.observables.MathObservable;
import java.util.*;
@@ -68,7 +65,7 @@
@Override
- public long sumAllCollections(ApplicationScope applicationScope) {
+ public long getApplicationSize(ApplicationScope applicationScope) {
final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope);
EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy);
GraphManager graphManager = graphManagerFactory.createEdgeManager(applicationScope);
@@ -83,7 +80,7 @@
}
@Override
- public Map<String, Long> sumEachCollection(ApplicationScope applicationScope) {
+ public Map<String, Long> getEachCollectionSize(ApplicationScope applicationScope) {
final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope);
EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy);
GraphManager graphManager = graphManagerFactory.createEdgeManager(applicationScope);
@@ -103,15 +100,15 @@
}
@Override
- public long sum(ApplicationScope applicationScope, SearchEdge edge) {
+ public long getSize(ApplicationScope applicationScope, SearchEdge edge) {
final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope);
EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy);
return entityIndex.getEntitySize(edge);
}
@Override
- public long getCollectionSum(final ApplicationScope applicationScope, final String collectionName) {
- return sum(applicationScope, CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), collectionName));
+ public long getCollectionSize(final ApplicationScope applicationScope, final String collectionName) {
+ return getSize(applicationScope, CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), collectionName));
}
}
diff --git a/stack/core/src/main/java/org/apache/usergrid/count/SimpleBatcher.java b/stack/core/src/main/java/org/apache/usergrid/count/SimpleBatcher.java
index d5a6bf2..b72f544 100644
--- a/stack/core/src/main/java/org/apache/usergrid/count/SimpleBatcher.java
+++ b/stack/core/src/main/java/org/apache/usergrid/count/SimpleBatcher.java
@@ -24,7 +24,7 @@
/**
- * A simple Batcher implementation that keeps a sum of the number of {@link Count} operations which have been applied.
+ * A simple Batcher implementation that keeps a getSize of the number of {@link Count} operations which have been applied.
* Counters are aggregated by name.
*
* @author zznate
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/AggregationServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/AggregationServiceTest.java
index d7d1ffa..9f1c9a4 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/AggregationServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/AggregationServiceTest.java
@@ -51,15 +51,15 @@
this.app.refreshIndex();
Thread.sleep(500);
- long sum = aggregationService.sumAllCollections(applicationScope);
+ long sum = aggregationService.getApplicationSize(applicationScope);
Assert.assertTrue( sum >= 0 );
Assert.assertTrue(sum > (entity1.getSize() + entity2.getSize()));
- long sum1 = aggregationService.sum(applicationScope,CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(),"tests"));
+ long sum1 = aggregationService.getSize(applicationScope, CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), "tests"));
Assert.assertEquals(sum1, entity1.getSize());
- long sum2 = aggregationService.sum(applicationScope, CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), "test2s"));
+ long sum2 = aggregationService.getSize(applicationScope, CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), "test2s"));
Assert.assertEquals(sum2, entity2.getSize());
props = new HashMap<>();
@@ -68,12 +68,12 @@
Entity entity3 = this.app.getEntityManager().create("test", props);
this.app.refreshIndex();
- long sum3 = aggregationService.sum(applicationScope, CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), "tests"));
- Assert.assertEquals(sum3, entity1.getSize()+entity3.getSize());
+ long sum3 = aggregationService.getSize(applicationScope, CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), "tests"));
+ Assert.assertEquals(sum3, entity1.getSize() + entity3.getSize());
- Map<String,Long> sumEach = aggregationService.sumEachCollection(applicationScope);
+ Map<String,Long> sumEach = aggregationService.getEachCollectionSize(applicationScope);
Assert.assertTrue(sumEach.containsKey("tests") && sumEach.containsKey("test2s"));
- Assert.assertEquals(sum3, (long)sumEach.get("tests"));
+ Assert.assertEquals(sum3, (long) sumEach.get("tests"));
}
}
diff --git a/stack/loadtests/README.md b/stack/loadtests/README.md
index 74f93bf..802c286 100644
--- a/stack/loadtests/README.md
+++ b/stack/loadtests/README.md
@@ -30,6 +30,9 @@
###runGetEntitiesByNameSequential.sh
Retrieves entities one by one via name (prefix + entity number).
+###runGetEntitiesByUuidTest.sh
+Retrieves entities via UUID from a CSV file. Control the CSV feed pattern (circular=sequential, random) using the csvFeedPattern configuration
+
###runLoadEntities.sh
Creates entities in order via name (prefix + entity number).
@@ -42,15 +45,15 @@
###runLoadSortableEntities.sh
Creates sortable entities in order via name (prefix + entity number).
+###runOrgAppSetup.sh
+Runs the organization and app setup without doing a test.
+
###runRandomEntityByNameQueryTest.sh
Retrieves random entities via name (prefix + entity number) using queries.
###runRandomEntityByNameTest.sh
Retrieves random entities via name (prefix + entity number).
-###runRandomEntityByUuidTest.sh
-Retrieves random entities via UUID from a CSV file.
-
###runUpdateEntities.sh
Updates entities in order via name (prefix + entity number).
@@ -74,7 +77,7 @@
* createOrg (**false**) - create the organization specified by the org configuration item (will continue if the org already exists)
* createApp (**false**) - create the application specified by the app configuration item (will continue if the app already exists)
* loadEntities (**false**) - load entities as part of setup, instead of as part of the test
-* scenarioType (**"nameRandomInfinite"**, "uuidRandomInfinite", "getByNameSequential", "getAllByCursor", "loadEntities", "updateEntities", "deleteEntities", "auditGetCollectionEntities", "auditVerifyCollectionEntities") - type of scenario to run, more details in test scripts section below
+* scenarioType (**"nameRandomInfinite"**, "uuidInfinite", "getByNameSequential", "getAllByCursor", "loadEntities", "updateEntities", "deleteEntities", "auditGetCollectionEntities", "auditVerifyCollectionEntities") - type of scenario to run, more details in test scripts section
* rampUsers (**0**) - number of users to inject during the ramp up time
* rampTime (**0**) - duration in seconds of the ramp up time
* constantUsersPerSec (**0**) - number of users per second to inject during the constant injection time (decimal ok)
@@ -110,6 +113,8 @@
* injectionList (**"rampUsers(10,60)"**) - custom injection pattern for CustomInjectionSimulation (discussed below)
* printFailedRequests (**true**) - prints the request and response on the console for failed requests (those that fail more than *retryCount* times)
* getViaQuery (**false**) - retrieve entities via query instead of via name or uuid
+* queryParams (**""**) - additional query parameters (currently used for get by entity or by name)
+* csvFeedPattern (**"random"**) - pattern to use when feeding from a CSV ("random" is random, "circular" goes through CSV sequentially and restarts from beginning when it reaches the end)
The following settings are currently not used (were used by deprecated tests, but may be valid in the future):
diff --git a/stack/loadtests/runRandomEntityByUuidTest.sh b/stack/loadtests/runGetEntitiesByUuid.sh
similarity index 88%
rename from stack/loadtests/runRandomEntityByUuidTest.sh
rename to stack/loadtests/runGetEntitiesByUuid.sh
index 01f4fe5..545ab0d 100755
--- a/stack/loadtests/runRandomEntityByUuidTest.sh
+++ b/stack/loadtests/runGetEntitiesByUuid.sh
@@ -39,14 +39,18 @@
#CONSTANT_USERS_PER_SEC=
#CONSTANT_USERS_DURATION=
+#CSV_FEED_PATTERN=circular
+CSV_FEED_PATTERN=random
+
die() { echo "$@" 1>&2 ; exit 1; }
-[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) UUID_FILENAME [QUERY_PARAMS]"
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) UUID_FILENAME [[CSV_FEED_PATTERN] QUERY_PARAMS]"
RAMP_USERS="$1"
RAMP_TIME="$2"
UUID_FILENAME="$3"
-[ "$#" -ge 4 ] && QUERY_PARAMS="$4"
+[ "$#" -ge 4 ] && CSV_FEED_PATTERN="$4"
+[ "$#" -ge 5 ] && QUERY_PARAMS="$5"
shift $#
@@ -83,5 +87,6 @@
-DuuidFilename=${UUID_FILENAME} \
-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
-DqueryParams=${QUERY_PARAMS} \
+-DcsvFeedPattern=${CSV_FEED_PATTERN} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
index 6647549..df0b325 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
@@ -80,6 +80,7 @@
val MultiPropertySizeInK = "multiPropertySizeInK"
val EntityNumberProperty = "entityNumberProperty"
val QueryParams = "queryParams"
+ val CsvFeedPattern = "csvFeedPattern"
val Values = Seq(Org,App,AdminUser,AdminPassword,BaseUrl,AuthType,TokenType,SkipSetup,CreateOrg,CreateApp,LoadEntities,
ScenarioType,RampUsers,ConstantUsersPerSec,ConstantUsersDuration,UserSeed,AppUser,AppUserPassword,NumEntities,
@@ -88,7 +89,7 @@
OrgCreationName,OrgCreationEmail,OrgCreationPassword,UpdateProperty,UpdateValue,EntityWorkerCount,EntityWorkerNum,
UuidFilename,AuditUuidFilename,FailedUuidFilename,SandboxCollection,PurgeUsers,RetryCount,LaterThanTimestamp,
EntityProgressCount,InjectionList,PrintFailedRequests,GetViaQuery,MultiPropertyPrefix,MultiPropertyCount,
- MultiPropertySizeInK,EntityNumberProperty,QueryParams)
+ MultiPropertySizeInK,EntityNumberProperty,QueryParams,CsvFeedPattern)
def isValid(str: String): Boolean = {
Values.contains(str)
@@ -156,6 +157,7 @@
case MultiPropertySizeInK => 1
case EntityNumberProperty => ""
case QueryParams => ""
+ case CsvFeedPattern => org.apache.usergrid.enums.CsvFeedPatternType.Random
}
} else {
null
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/CsvFeedPatternType.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/CsvFeedPatternType.scala
new file mode 100644
index 0000000..6c1ab8d
--- /dev/null
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/CsvFeedPatternType.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.enums
+
+/**
+ * Created by mdunker on 8/31/15.
+ */
+object CsvFeedPatternType {
+ val Random = "random"
+ val Circular = "circular"
+
+ val Values = Seq(Random,Circular)
+ def isValid(str: String): Boolean = {
+ Values.contains(str)
+ }
+}
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index 38a70ff..0ab2cd6 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -20,7 +20,7 @@
import io.gatling.core.feeder.RecordSeqFeederBuilder
import io.gatling.http.Predef._
import org.apache.usergrid.datagenerators.FeederGenerator
-import org.apache.usergrid.enums.{EndConditionType, AuthType}
+import org.apache.usergrid.enums.{CsvFeedPatternType, EndConditionType, AuthType}
import org.apache.usergrid.helpers.Extractors._
import org.apache.usergrid.helpers.{Headers, Utils}
import org.apache.usergrid.settings.Settings
@@ -58,8 +58,11 @@
}
def uuidFeeder(): RecordSeqFeederBuilder[String] = {
-
- csv(Settings.feedUuidFilename).random
+ if (Settings.csvFeedPattern == CsvFeedPatternType.Circular) {
+ csv(Settings.feedUuidFilename).circular
+ } else {
+ csv(Settings.feedUuidFilename).random
+ }
}
/*
@@ -160,7 +163,7 @@
.check(status.is(200))
)
- val getRandomEntitiesByUuid = scenario("Get entities by uuid randomly")
+ val getRandomEntitiesByUuid = scenario("Get entities by uuid")
.exec(injectTokenIntoSession())
.exec(injectAuthType())
.doIfOrElse(_ => Settings.endConditionType == EndConditionType.MinutesElapsed) {
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index 0a800cc..e9dd2f2 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -180,6 +180,7 @@
val getViaQuery:Boolean = initBoolSetting(ConfigProperties.GetViaQuery)
private val queryParamConfig = initStrSetting(ConfigProperties.QueryParams)
val queryParamMap: Map[String,String] = mapFromQueryParamConfigString(queryParamConfig)
+ val csvFeedPattern = initStrSetting(ConfigProperties.CsvFeedPattern)
val multiPropertyPrefix = initStrSetting(ConfigProperties.MultiPropertyPrefix)
val multiPropertyCount:Int = initIntSetting(ConfigProperties.MultiPropertyCount)
@@ -454,6 +455,7 @@
} else {
println(s"SearchLimit:$searchLimit SearchQuery:$searchQuery")
}
+ if (queryParamConfig != "") println(s"Extra query params: $queryParamConfig")
println()
println(s"Overall: NumEntities:$totalNumEntities Seed:$overallEntitySeed Workers:$entityWorkerCount")
println(s"Worker: NumEntities:$numEntities Seed:$entitySeed WorkerNum:$entityWorkerNum")
@@ -462,7 +464,10 @@
println(s"Constant: UsersPerSec:$constantUsersPerSec Time:$constantUsersDuration")
println(s"EndCondition:$endConditionStr")
println()
- if (feedUuids) println(s"Feed CSV: $feedUuidFilename")
+ if (feedUuids) {
+ println(s"Feed CSV: $feedUuidFilename")
+ println(s"Feed pattern: $csvFeedPattern")
+ }
if (feedAuditUuids) println(s"Audit Feed CSV: $feedAuditUuidFilename")
if (captureUuids) println(s"Capture CSV:$captureUuidFilename")
if (captureAuditUuids) {
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
index df98b43..be8af2d 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
@@ -162,7 +162,7 @@
@RequireOrganizationAccess
@GET
- @Path("size")
+ @Path("_size")
public JSONWithPadding getApplicationSize(
@Context UriInfo ui, @QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
@@ -173,16 +173,16 @@
Map<String,Object> map = new HashMap<>();
Map<String,Object> innerMap = new HashMap<>();
Map<String,Object> sumMap = new HashMap<>();
- sumMap.put("sum",size);
- innerMap.put("application",sumMap);
- map.put("aggregation",innerMap);
+ innerMap.put("application",size);
+ sumMap.put("size",innerMap);
+ map.put("aggregation", sumMap);
response.setMetadata(map);
return new JSONWithPadding( response, callback );
}
@RequireOrganizationAccess
@GET
- @Path("size/{collection_name}")
+ @Path("{collection_name}/_size")
public JSONWithPadding getCollectionSize(
@Context UriInfo ui,
@PathParam( "collection_name" ) String collection_name,
@@ -190,13 +190,31 @@
throws Exception {
ApiResponse response = createApiResponse();
response.setAction("get collection size for all entities");
- long size = management.getCollectionSize(this.applicationId ,collection_name);
+ long size = management.getCollectionSize(this.applicationId, collection_name);
Map<String,Object> map = new HashMap<>();
- Map<String,Object> innerMap = new HashMap<>();
Map<String,Object> sumMap = new HashMap<>();
- sumMap.put("sum",size);
- innerMap.put(collection_name,sumMap);
- map.put("aggregation",innerMap);
+ Map<String,Object> innerMap = new HashMap<>();
+ innerMap.put(collection_name,size);
+ sumMap.put("size",innerMap);
+ map.put("aggregation",sumMap);
+ response.setMetadata(map);
+ return new JSONWithPadding( response, callback );
+ }
+
+ @RequireOrganizationAccess
+ @GET
+ @Path("collections/_size")
+ public JSONWithPadding getEachCollectionSize(
+ @Context UriInfo ui,
+ @QueryParam("callback") @DefaultValue("callback") String callback )
+ throws Exception {
+ ApiResponse response = createApiResponse();
+ response.setAction("get collection size for all entities");
+ Map<String,Long> sizes = management.getEachCollectionSize(this.applicationId);
+ Map<String,Object> map = new HashMap<>();
+ Map<String,Object> sumMap = new HashMap<>();
+ sumMap.put("size",sizes);
+ map.put("aggregation",sumMap);
response.setMetadata(map);
return new JSONWithPadding( response, callback );
}
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index 99d2089..7f86ad1 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -314,18 +314,22 @@
final String appname = clientSetup.getAppName();
this.app().collection("testCollection").post(new Entity().chainPut("name","test"));
refreshIndex();
- Entity size = management().orgs().org( clientSetup.getOrganizationName() ).app().addToPath(appname).addToPath("size").get();
- Entity rolesSize = management().orgs().org(clientSetup.getOrganizationName()).app().addToPath(appname).addToPath("size/roles").get();
+ Entity size = management().orgs().org( clientSetup.getOrganizationName() ).app().addToPath(appname).addToPath("_size").get();
+ Entity rolesSize = management().orgs().org(clientSetup.getOrganizationName()).app().addToPath(appname).addToPath("roles/_size").get();
+ Entity collectionsSize = management().orgs().org(clientSetup.getOrganizationName()).app().addToPath(appname).addToPath("collections/_size").get();
+
assertTrue(size != null);
assertTrue(rolesSize != null);
- int sum = (int)((LinkedHashMap)((LinkedHashMap)size.metadata().get("aggregation")).get("application")).get("sum");
- int sumRoles = (int)((LinkedHashMap)((LinkedHashMap)rolesSize.metadata().get("aggregation")).get("roles")).get("sum");
+ int sum = (int)((LinkedHashMap)((LinkedHashMap)size.metadata().get("aggregation")).get("size")).get("application");
+ int sumRoles = (int)((LinkedHashMap)((LinkedHashMap)rolesSize.metadata().get("aggregation")).get("size")).get("roles");
+ int sumRoles2 = (int)((LinkedHashMap)((LinkedHashMap)collectionsSize.metadata().get("aggregation")).get("size")).get("roles");
assertTrue(size != null);
assertTrue(rolesSize != null);
- assertNotEquals(sum,sumRoles);
- assertTrue(sum>sumRoles);
+ assertNotEquals(sum, sumRoles);
+ assertTrue(sum > sumRoles);
+ assertTrue(sumRoles == sumRoles2);
}
@Test
diff --git a/stack/scripts/migrate_entity_data.py b/stack/scripts/migrate_entity_data.py
index 13c1b41..66a9dd0 100644
--- a/stack/scripts/migrate_entity_data.py
+++ b/stack/scripts/migrate_entity_data.py
@@ -53,6 +53,7 @@
# Version expected in status response post-migration for entity and app-info data
TARGET_VERSION = 2
TARGET_MIGRATION_SYSTEM_VERSION = 1
+TARGET_INDEX_MAPPING_VERSION = 1
# Set an interval (in seconds) for checking if re-index and/or migration has finished
STATUS_INTERVAL_SECONDS = 2
@@ -61,6 +62,7 @@
PLUGIN_MIGRATION_SYSTEM = 'migration-system'
PLUGIN_APPINFO = 'appinfo-migration'
PLUGIN_ENTITYDATA = 'collections-entity-data'
+PLUGIN_INDEX_MAPPING = 'index_mapping_migration'
@@ -132,7 +134,7 @@
migration_system_updated = self.is_migration_system_updated()
if not migration_system_updated:
- self.logger.info('Migration system needs to updated. Updating migration system..')
+ self.logger.info('Migration system needs to be updated. Updating migration system..')
self.start_migration_system_update()
while not migration_system_updated:
time.sleep(STATUS_INTERVAL_SECONDS)
@@ -163,6 +165,18 @@
else:
self.logger.info('Full Data Migration previously ran... skipping AppInfo migration.')
+ # We need to check and roll index mapping version to 1 if not already there
+ index_mapping_updated = self.is_index_mapping_updated()
+
+ if not index_mapping_updated:
+ self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
+ self.start_index_mapping_migration()
+ while not index_mapping_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ index_mapping_updated = self.is_index_mapping_updated()
+ if index_mapping_updated:
+ break
+
# Perform system re-index (it will grab date from input if provided)
job = self.start_reindex()
self.metrics['reindex_start'] = get_current_time()
@@ -241,6 +255,16 @@
self.logger.error('Failed to start migration, %s', e)
exit_on_error(str(e))
+ def start_index_mapping_migration(self):
+ try:
+ migrateUrl = self.get_migration_url() + '/' + PLUGIN_INDEX_MAPPING
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
def start_appinfo_migration(self):
try:
migrateUrl = self.get_migration_url() + '/' + PLUGIN_APPINFO
@@ -325,6 +349,22 @@
migration_system_version)
return False
+ def is_index_mapping_updated(self):
+ status = self.check_data_migration_status()
+ if status is not None:
+ index_mapping_version = status['data'][PLUGIN_INDEX_MAPPING]
+
+ if index_mapping_version == TARGET_INDEX_MAPPING_VERSION:
+ self.logger.info('Index Mapping CURRENT, %s=[%s]',
+ PLUGIN_INDEX_MAPPING,
+ index_mapping_version)
+ return True
+ else:
+ self.logger.info('Index Mapping OLD, %s=[%s]',
+ PLUGIN_INDEX_MAPPING,
+ index_mapping_version)
+ return False
+
def check_data_migration_status(self):
try:
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index c822057..43c1b30 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -336,4 +336,6 @@
long getApplicationSize(final UUID applicationId);
long getCollectionSize(final UUID applicationId, final String collectionName);
+
+ Map<String,Long> getEachCollectionSize(final UUID applicationId);
}
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index e36e31c..73f18b1 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -287,7 +287,7 @@
boolean superuser_enabled = getBooleanProperty( PROPERTIES_SYSADMIN_LOGIN_ALLOWED );
String superuser_username = properties.getProperty( PROPERTIES_SYSADMIN_LOGIN_NAME );
String superuser_email = properties.getProperty( PROPERTIES_SYSADMIN_LOGIN_EMAIL );
- String superuser_password = properties.getProperty( PROPERTIES_SYSADMIN_LOGIN_PASSWORD );
+ String superuser_password = properties.getProperty(PROPERTIES_SYSADMIN_LOGIN_PASSWORD);
if ( !anyNull( superuser_username, superuser_email, superuser_password ) ) {
UserInfo user = this.getAdminUserByUsername( superuser_username );
@@ -301,7 +301,7 @@
}
else {
System.out.println(
- "Missing values for superuser account, check properties. Skipping superuser account setup..." );
+ "Missing values for superuser account, check properties. Skipping superuser account setup...");
}
}
@@ -389,7 +389,7 @@
@Override
public ServiceResults getAdminUserActivity( UserInfo user ) throws Exception {
- ServiceManager sm = smf.getServiceManager( smf.getManagementAppId() );
+ ServiceManager sm = smf.getServiceManager(smf.getManagementAppId());
return sm.newRequest( ServiceAction.GET, parameters( "users", user.getUuid(), "feed" ) ).execute();
}
@@ -680,7 +680,7 @@
@Override
public BiMap<UUID, String> getOrganizations() throws Exception {
- List<OrganizationInfo> orgs = getOrganizations( null, 10000 );
+ List<OrganizationInfo> orgs = getOrganizations(null, 10000);
return buildOrgBiMap( orgs );
}
@@ -696,7 +696,7 @@
@Override
public OrganizationInfo getOrganizationInfoFromAccessToken( String token ) throws Exception {
- Entity entity = getEntityFromAccessToken( token, null, ORGANIZATION );
+ Entity entity = getEntityFromAccessToken(token, null, ORGANIZATION);
if ( entity == null ) {
return null;
}
@@ -711,7 +711,7 @@
return null;
}
- EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
+ EntityManager em = emf.getEntityManager(smf.getManagementAppId());
EntityRef ref = em.getAlias( Group.ENTITY_TYPE, organizationName );
if ( ref == null ) {
return null;
@@ -820,17 +820,17 @@
@Override
public UserInfo createAdminFrom( User user, String password ) throws Exception {
- return doCreateAdmin( user,
- encryptionService.defaultEncryptedCredentials( password, user.getUuid(), smf.getManagementAppId() ),
- encryptionService.plainTextCredentials( mongoPassword( user.getUsername(), password ), user.getUuid(),
- smf.getManagementAppId() ) );
+ return doCreateAdmin(user,
+ encryptionService.defaultEncryptedCredentials(password, user.getUuid(), smf.getManagementAppId()),
+ encryptionService.plainTextCredentials(mongoPassword(user.getUsername(), password), user.getUuid(),
+ smf.getManagementAppId()));
}
@Override
public UserInfo createAdminUser( String username, String name, String email, String password, boolean activated,
boolean disabled ) throws Exception {
- return createAdminUser( username, name, email, password, activated, disabled, null );
+ return createAdminUser(username, name, email, password, activated, disabled, null);
}
@@ -838,7 +838,7 @@
public UserInfo createAdminUser( String username, String name, String email, String password, boolean activated,
boolean disabled, Map<String, Object> userProperties ) throws Exception {
- if ( !validateAdminInfo( username, name, email, password ) ) {
+ if ( !validateAdminInfo(username, name, email, password) ) {
return null;
}
return createAdminUserInternal( username, name, email, password, activated, disabled, userProperties );
@@ -935,10 +935,10 @@
List<UserInfo> users = new ArrayList<UserInfo>();
- EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
+ EntityManager em = emf.getEntityManager(smf.getManagementAppId());
Results results =
- em.getCollection( new SimpleEntityRef( Group.ENTITY_TYPE, organizationId ), "users", null, 10000,
- Level.ALL_PROPERTIES, false );
+ em.getCollection(new SimpleEntityRef(Group.ENTITY_TYPE, organizationId), "users", null, 10000,
+ Level.ALL_PROPERTIES, false);
for ( Entity entity : results.getEntities() ) {
users.add( getUserInfo( smf.getManagementAppId(), entity ) );
}
@@ -1003,7 +1003,7 @@
return null;
}
- return getUserEntityByIdentifier( smf.getManagementAppId(), Identifier.fromEmail( email ) );
+ return getUserEntityByIdentifier(smf.getManagementAppId(), Identifier.fromEmail(email));
}
@@ -1012,14 +1012,14 @@
if ( email == null ) {
return null;
}
- return getUserInfo( smf.getManagementAppId(),
- getUserEntityByIdentifier( smf.getManagementAppId(), Identifier.fromEmail( email ) ) );
+ return getUserInfo(smf.getManagementAppId(),
+ getUserEntityByIdentifier(smf.getManagementAppId(), Identifier.fromEmail(email)));
}
public User getUserEntityByIdentifier( UUID applicationId, Identifier identifier ) throws Exception {
EntityManager em = emf.getEntityManager( applicationId );
- return em.get( em.getUserByIdentifier( identifier ), User.class );
+ return em.get(em.getUserByIdentifier(identifier), User.class);
}
@@ -1028,8 +1028,8 @@
if ( username == null ) {
return null;
}
- return getUserInfo( smf.getManagementAppId(),
- getUserEntityByIdentifier( smf.getManagementAppId(), Identifier.fromName( username ) ) );
+ return getUserInfo(smf.getManagementAppId(),
+ getUserEntityByIdentifier(smf.getManagementAppId(), Identifier.fromName(username)));
}
@@ -1038,14 +1038,14 @@
if ( id == null ) {
return null;
}
- return getUserEntityByIdentifier( smf.getManagementAppId(), Identifier.fromUUID( id ) );
+ return getUserEntityByIdentifier(smf.getManagementAppId(), Identifier.fromUUID(id));
}
@Override
public UserInfo getAdminUserByUuid( UUID id ) throws Exception {
- return getUserInfo( smf.getManagementAppId(),
- getUserEntityByIdentifier( smf.getManagementAppId(), Identifier.fromUUID( id ) ) );
+ return getUserInfo(smf.getManagementAppId(),
+ getUserEntityByIdentifier(smf.getManagementAppId(), Identifier.fromUUID(id)));
}
@@ -1127,7 +1127,7 @@
throw new IncorrectPasswordException( "Old password does not match" );
}
- setAdminUserPassword( userId, newPassword );
+ setAdminUserPassword(userId, newPassword);
}
@@ -1204,7 +1204,7 @@
Results orgResults = em.getCollection( new SimpleEntityRef( User.ENTITY_TYPE, userId ),
Schema.COLLECTION_GROUPS, null, 10000, Level.REFS, false );
- logger.debug( " orgResults.size() = " + orgResults.size() );
+ logger.debug(" orgResults.size() = " + orgResults.size());
for ( EntityRef orgRef : orgResults.getRefs() ) {
Map properties = em.getDictionaryAsMap( orgRef, ORGANIZATION_PROPERTIES_DICTIONARY );
@@ -1308,7 +1308,7 @@
public String getTokenForPrincipal( TokenCategory token_category, String token_type, UUID applicationId,
AuthPrincipalType principal_type, UUID id, long duration ) throws Exception {
- if ( anyNull( token_category, applicationId, principal_type, id ) ) {
+ if ( anyNull(token_category, applicationId, principal_type, id) ) {
return null;
}
@@ -1361,7 +1361,7 @@
AuthPrincipalType expected_principal_type ) throws Exception {
AuthPrincipalInfo principal =
- getPrincipalFromAccessToken( token, expected_token_type, expected_principal_type );
+ getPrincipalFromAccessToken(token, expected_token_type, expected_principal_type);
if ( principal == null ) {
return null;
}
@@ -1376,8 +1376,8 @@
principal.getApplicationId() != null
? principal.getApplicationId() : smf.getManagementAppId() );
- Entity entity = em.get( new SimpleEntityRef(
- principal.getType().getEntityType(), principal.getUuid()));
+ Entity entity = em.get(new SimpleEntityRef(
+ principal.getType().getEntityType(), principal.getUuid()));
return entity;
}
@@ -1406,7 +1406,7 @@
*/
@Override
public void revokeAccessTokensForAdminUser( UUID userId ) throws Exception {
- revokeTokensForPrincipal( ADMIN_USER, smf.getManagementAppId(), userId );
+ revokeTokensForPrincipal(ADMIN_USER, smf.getManagementAppId(), userId);
}
@@ -1421,21 +1421,21 @@
throw new TokenException( "Could not match token : " + token );
}
- tokens.revokeToken( token );
+ tokens.revokeToken(token);
}
@Override
public Entity getAdminUserEntityFromAccessToken( String token ) throws Exception {
- Entity user = getEntityFromAccessToken( token, null, ADMIN_USER );
+ Entity user = getEntityFromAccessToken(token, null, ADMIN_USER);
return user;
}
@Override
public UserInfo getAdminUserInfoFromAccessToken( String token ) throws Exception {
- Entity user = getAdminUserEntityFromAccessToken( token );
+ Entity user = getAdminUserEntityFromAccessToken(token);
return new UserInfo( smf.getManagementAppId(), user.getProperties() );
}
@@ -1487,7 +1487,7 @@
@Override
public Long getLastAdminPasswordChange( UUID userId ) throws Exception {
- CredentialsInfo ci = readUserPasswordCredentials( smf.getManagementAppId(), userId, User.ENTITY_TYPE );
+ CredentialsInfo ci = readUserPasswordCredentials(smf.getManagementAppId(), userId, User.ENTITY_TYPE);
return ci.getCreated();
}
@@ -1568,8 +1568,8 @@
}
EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
- em.addToCollection( new SimpleEntityRef( Group.ENTITY_TYPE, organization.getUuid() ), "users",
- new SimpleEntityRef( User.ENTITY_TYPE, user.getUuid() ) );
+ em.addToCollection(new SimpleEntityRef(Group.ENTITY_TYPE, organization.getUuid()), "users",
+ new SimpleEntityRef(User.ENTITY_TYPE, user.getUuid()));
if ( email ) {
sendAdminUserInvitedEmail( user, organization );
@@ -1596,8 +1596,8 @@
throw new UnableToLeaveOrganizationException( "Organizations must have at least one member." );
}
- em.removeFromCollection( new SimpleEntityRef( Group.ENTITY_TYPE, organizationId ), "users",
- new SimpleEntityRef( User.ENTITY_TYPE, userId ) );
+ em.removeFromCollection(new SimpleEntityRef(Group.ENTITY_TYPE, organizationId), "users",
+ new SimpleEntityRef(User.ENTITY_TYPE, userId));
}
@@ -1721,7 +1721,15 @@
AggregationServiceFactory aggregationServiceFactory = injector.getInstance(AggregationServiceFactory.class);
AggregationService aggregationService = aggregationServiceFactory.getAggregationService();
ApplicationScope applicationScope =CpNamingUtils.getApplicationScope(applicationId);
- return aggregationService.sumAllCollections(applicationScope);
+ return aggregationService.getApplicationSize(applicationScope);
+ }
+
+ @Override
+ public Map<String,Long> getEachCollectionSize(final UUID applicationId) {
+ AggregationServiceFactory aggregationServiceFactory = injector.getInstance(AggregationServiceFactory.class);
+ AggregationService aggregationService = aggregationServiceFactory.getAggregationService();
+ ApplicationScope applicationScope =CpNamingUtils.getApplicationScope(applicationId);
+ return aggregationService.getEachCollectionSize(applicationScope);
}
@Override
@@ -1729,10 +1737,11 @@
AggregationServiceFactory aggregationServiceFactory = injector.getInstance(AggregationServiceFactory.class);
AggregationService aggregationService = aggregationServiceFactory.getAggregationService();
ApplicationScope applicationScope =CpNamingUtils.getApplicationScope(applicationId);
- return aggregationService.sum(applicationScope,CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(),collectionName));
+ return aggregationService.getSize(applicationScope, CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), collectionName));
}
+
@Override
public OrganizationInfo getOrganizationForApplication( UUID applicationInfoId ) throws Exception {
