Merge pull request #42 from shawnfeldman/master
adding generic test account
diff --git a/sdks/nodejs/lib/usergrid.js b/sdks/nodejs/lib/usergrid.js
index 3be7c77..59cb843 100755
--- a/sdks/nodejs/lib/usergrid.js
+++ b/sdks/nodejs/lib/usergrid.js
@@ -2207,11 +2207,198 @@
}
return tail.join("&");
}
+/*
+ * A class to model a Usergrid event.
+ *
+ * @constructor
+ * @param {object} options {timestamp:0, category:'value', counters:{name : value}}
+ * @returns {callback} callback(err, event)
+ */
+Usergrid.Counter = function(options, callback) {
+ var self=this;
+ this._client = options.client;
+ this._data = options.data || {};
+ this._data.category = options.category||"UNKNOWN";
+ this._data.timestamp = options.timestamp||0;
+ this._data.type = "events";
+ this._data.counters=options.counters||{};
+ if(typeof(callback) === 'function') {
+ callback.call(self, false, self);
+ }
+ //this.save(callback);
+};
+var COUNTER_RESOLUTIONS=[
+ 'all', 'minute', 'five_minutes', 'half_hour',
+ 'hour', 'six_day', 'day', 'week', 'month'
+];
+/*
+ * Inherit from Usergrid.Entity.
+ * Note: This only accounts for data on the group object itself.
+ * You need to use add and remove to manipulate group membership.
+ */
+Usergrid.Counter.prototype = new Usergrid.Entity();
+
+/*
+ * overrides Entity.prototype.fetch. Returns all data for counters
+ * associated with the object as specified in the constructor
+ *
+ * @public
+ * @method increment
+ * @param {function} callback
+ * @returns {callback} callback(err, event)
+ */
+Usergrid.Counter.prototype.fetch=function(callback){
+ this.getData({}, callback);
+}
+/*
+ * increments the counter for a specific event
+ *
+ * options object: {name: counter_name}
+ *
+ * @public
+ * @method increment
+ * @params {object} options
+ * @param {function} callback
+ * @returns {callback} callback(err, event)
+ */
+Usergrid.Counter.prototype.increment=function(options, callback){
+ var self=this,
+ name=options.name,
+ value=options.value;
+ if(!name){
+ if(typeof(callback) === 'function') {
+ return callback.call(self, true, "'value' for increment, decrement must be a number");
+ }
+ }else if(isNaN(value)){
+ if(typeof(callback) === 'function') {
+ return callback.call(self, true, "'value' for increment, decrement must be a number");
+ }
+ }else{
+ self._data.counters[name]=(parseInt(value))||1;
+ return self.save(callback);
+ }
+};
+/*
+ * decrements the counter for a specific event
+ *
+ * options object: {name: counter_name}
+ *
+ * @public
+ * @method decrement
+ * @params {object} options
+ * @param {function} callback
+ * @returns {callback} callback(err, event)
+ */
+
+Usergrid.Counter.prototype.decrement=function(options, callback){
+ var self=this,
+ name=options.name,
+ value=options.value;
+ self.increment({name:name, value:-((parseInt(value))||1)}, callback);
+};
+/*
+ * resets the counter for a specific event
+ *
+ * options object: {name: counter_name}
+ *
+ * @public
+ * @method reset
+ * @params {object} options
+ * @param {function} callback
+ * @returns {callback} callback(err, event)
+ */
+
+Usergrid.Counter.prototype.reset=function(options, callback){
+ var self=this,
+ name=options.name;
+ self.increment({name:name, value:0}, callback);
+};
+
+/*
+ * gets data for one or more counters over a given
+ * time period at a specified resolution
+ *
+ * options object: {
+ * counters: ['counter1', 'counter2', ...],
+ * start: epoch timestamp or ISO date string,
+ * end: epoch timestamp or ISO date string,
+ * resolution: one of ('all', 'minute', 'five_minutes', 'half_hour', 'hour', 'six_day', 'day', 'week', or 'month')
+ * }
+ *
+ * @public
+ * @method getData
+ * @params {object} options
+ * @param {function} callback
+ * @returns {callback} callback(err, event)
+ */
+Usergrid.Counter.prototype.getData=function(options, callback){
+ var start_time,
+ end_time,
+ start=options.start||0,
+ end=options.end||Date.now(),
+ resolution=(options.resolution||'all').toLowerCase(),
+ counters=options.counters||Object.keys(this._data.counters),
+ res=(resolution||'all').toLowerCase();
+ if(COUNTER_RESOLUTIONS.indexOf(res)===-1){
+ res='all';
+ }
+ if(start){
+ switch(typeof start){
+ case "undefined":
+ start_time=0;
+ break;
+ case "number":
+ start_time=start;
+ break;
+ case "string":
+ start_time=(isNaN(start))?Date.parse(start):parseInt(start);
+ break;
+ default:
+ start_time=Date.parse(start.toString());
+ }
+ }
+ if(end){
+ switch(typeof end){
+ case "undefined":
+ end_time=Date.now();
+ break;
+ case "number":
+ end_time=end;
+ break;
+ case "string":
+ end_time=(isNaN(end))?Date.parse(end):parseInt(end);
+ break;
+ default:
+ end_time=Date.parse(end.toString());
+ }
+ }
+ var self=this;
+ //https://api.usergrid.com/yourorgname/sandbox/counters?counter=test_counter
+ var params=Object.keys(counters).map(function(counter){
+ return ["counter", encodeURIComponent(counters[counter])].join('=');
+ });
+ params.push('resolution='+res)
+ params.push('start_time='+String(start_time))
+ params.push('end_time='+String(end_time))
+
+ var endpoint="counters?"+params.join('&');
+ this._client.request({endpoint:endpoint}, function(err, data){
+ if(data.counters && data.counters.length){
+ data.counters.forEach(function(counter){
+ self._data.counters[counter.name]=counter.value||counter.values;
+ })
+ }
+ if(typeof(callback) === 'function') {
+ callback.call(self, err, data);
+ }
+ })
+};
exports.client = Usergrid.Client;
exports.entity = Usergrid.Entity;
exports.collection = Usergrid.Collection;
exports.group = Usergrid.Group;
+exports.counter = Usergrid.Counter;
exports.AUTH_CLIENT_ID = AUTH_CLIENT_ID;
exports.AUTH_APP_USER = AUTH_APP_USER;
exports.AUTH_NONE = AUTH_NONE;
diff --git a/sdks/nodejs/test.js b/sdks/nodejs/test.js
index c93754a..fabb27e 100755
--- a/sdks/nodejs/test.js
+++ b/sdks/nodejs/test.js
@@ -23,23 +23,12 @@
var _password = 'password2';
var _newpassword = 'password3';
- var client = new usergrid.client({
- orgName:'yourorgname',
- appName:'yourappname',
- authType:usergrid.AUTH_CLIENT_ID,
- clientId:'<your client id>',
- clientSecret:'<your client secret>',
- logging: false, //optional - turn on logging, off by default
- buildCurl: false //optional - turn on curl commands, off by default
- });
-
-
- var client = new usergrid.client({
- orgName:'yourorgname',
- appName:'sandbox',
- logging: true, //optional - turn on logging, off by default
- buildCurl: true //optional - turn on curl commands, off by default
- });
+var client = new usergrid.client({
+ orgName:'yourorgname',
+ appName:'sandbox',
+ logging: true, //optional - turn on logging, off by default
+ buildCurl: true //optional - turn on curl commands, off by default
+});
//call the runner function to start the process
@@ -186,6 +175,26 @@
notice('-----running step '+step+': clean up dogs');
cleanUpDogs(step, arg);
break;
+ case 35:
+ notice('-----running step '+step+': create counter');
+ counterCreate(step, arg);
+ break;
+ case 36:
+ notice('-----running step '+step+': reset counter');
+ counterReset(step, arg);
+ break;
+ case 37:
+ notice('-----running step '+step+': increment counter');
+ counterIncrement(step, arg);
+ break;
+ case 38:
+ notice('-----running step '+step+': decrement counter');
+ counterDecrement(step, arg);
+ break;
+ case 34:
+ notice('-----running step '+step+': fetch counter data');
+ counterFetch(step, arg);
+ break;
default:
notice('-----test complete!-----');
notice('Success count= ' + successCount);
@@ -998,3 +1007,54 @@
}
});
}
+//var counter;
+function counterCreate(step){
+ var counter = new usergrid.counter({client:client, data:{category:'mocha_test', timestamp:0, name:"test", counters:{test:0,test_counter:0}}}, function(err, data){
+ if (err) {
+ error('counter not removed');
+ } else {
+ success('counter created');
+ }
+ });
+ runner(step, counter);
+}
+function counterReset(step, counter){
+ counter.reset({name:'test'}, function(err, data){
+ if (err) {
+ error('counter not reset');
+ } else {
+ success('counter reset');
+ }
+ runner(step, counter);
+ });
+}
+function counterIncrement(step, counter){
+ counter.increment({name:'test', value:1}, function(err, data){
+ if (err) {
+ error('counter not incremented');
+ } else {
+ success('counter incremented');
+ }
+ runner(step, counter);
+ });
+}
+function counterDecrement(step, counter){
+ counter.decrement({name:'test', value:1}, function(err, data){
+ if (err) {
+ error('counter not decremented');
+ } else {
+ success('counter decremented');
+ }
+ runner(step, counter);
+ });
+}
+function counterFetch(step, counter){
+ counter.getData({resolution:'all', counters:['test', 'test_counter']}, function(err, data){
+ if (err) {
+ error('counter not fetched');
+ } else {
+ success('counter fetched');
+ }
+ runner(step, counter);
+ });
+}
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java
index a6b6268..6e166b6 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java
@@ -1047,7 +1047,7 @@
* @throws Exception the exception
*/
public IndexScanner getIdList( Keyspace ko, Object key, UUID start, UUID finish, int count, boolean reversed,
- IndexBucketLocator locator, UUID applicationId, String collectionName )
+ IndexBucketLocator locator, UUID applicationId, String collectionName, boolean keepFirst )
throws Exception {
if ( count <= 0 ) {
@@ -1058,9 +1058,12 @@
start = null;
}
+
+ final boolean skipFirst = start != null && !keepFirst;
+
IndexScanner scanner =
new IndexBucketScanner( this, locator, ENTITY_ID_SETS, applicationId, IndexType.COLLECTION, key, start,
- finish, reversed, count, collectionName );
+ finish, reversed, count, skipFirst, collectionName );
return scanner;
}
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java
index b5d77a3..5d9bbc8 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java
@@ -77,7 +77,7 @@
public class QueryProcessor {
- private static final int PAGE_SIZE = 1000;
+ public static final int PAGE_SIZE = 1000;
private static final Logger logger = LoggerFactory.getLogger( QueryProcessor.class );
private static final Schema SCHEMA = getDefaultSchema();
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java
index 4f565c7..577f803 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -1379,7 +1379,7 @@
IndexScanner scanner =
new IndexBucketScanner( cass, indexBucketLocator, ENTITY_INDEX, applicationId, IndexType.CONNECTION,
- keyPrefix, range[0], range[1], slice.isReversed(), pageSize, slice.getPropertyName() );
+ keyPrefix, range[0], range[1], slice.isReversed(), pageSize, slice.hasCursor(), slice.getPropertyName() );
return scanner;
}
@@ -1400,14 +1400,9 @@
Object keyPrefix = key( indexKey, slice.getPropertyName() );
- // we have a cursor, so the first record should be discarded
- if ( slice.hasCursor() ) {
- pageSize++;
- }
-
IndexScanner scanner =
new IndexBucketScanner( cass, indexBucketLocator, ENTITY_INDEX, applicationId, IndexType.COLLECTION,
- keyPrefix, range[0], range[1], slice.isReversed(), pageSize, collectionName );
+ keyPrefix, range[0], range[1], slice.isReversed(), pageSize, slice.hasCursor(), collectionName );
return scanner;
}
@@ -2128,14 +2123,13 @@
startId = UUID_PARSER.parse( slice.getCursor() ).getUUID();
}
- boolean skipFirst = node.isForceKeepFirst() ? false : slice.hasCursor();
IndexScanner indexScanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
key( headEntity.getUuid(), DICTIONARY_COLLECTIONS, collectionName ), startId, null,
queryProcessor.getPageSizeHint( node ), query.isReversed(), indexBucketLocator, applicationId,
- collectionName );
+ collectionName, node.isForceKeepFirst() );
- this.results.push( new SliceIterator( slice, indexScanner, UUID_PARSER, skipFirst ) );
+ this.results.push( new SliceIterator( slice, indexScanner, UUID_PARSER ) );
}
@@ -2267,10 +2261,6 @@
start = slice.getCursor();
}
- // we'll discard the first match, increase the size
- if ( start != null ) {
- size++;
- }
boolean skipFirst = node.isForceKeepFirst() ? false : slice.hasCursor();
@@ -2316,9 +2306,9 @@
IndexScanner connectionScanner =
new ConnectedIndexScanner( cass, dictionaryType, applicationId, entityIdToUse, connectionTypes,
- start, slice.isReversed(), size );
+ start, slice.isReversed(), size, skipFirst );
- this.results.push( new SliceIterator( slice, connectionScanner, connectionParser, skipFirst ) );
+ this.results.push( new SliceIterator( slice, connectionScanner, connectionParser ) );
}
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
index 265dc4d..378f2bd 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
@@ -34,7 +34,9 @@
import static org.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-/** @author tnine */
+/**
+ * @author tnine
+ */
public class ConnectedIndexScanner implements IndexScanner {
private final CassandraService cass;
@@ -44,24 +46,35 @@
private final String dictionaryType;
private final UUID entityId;
private final Iterator<String> connectionTypes;
+ private final boolean skipFirst;
- /** Pointer to our next start read */
+
+ /**
+ * Pointer to our next start read
+ */
private ByteBuffer start;
- /** Set to the original value to start scanning from */
+ /**
+ * Set to the original value to start scanning from
+ */
private ByteBuffer scanStart;
- /** Iterator for our results from the last page load */
+ /**
+ * Iterator for our results from the last page load
+ */
private LinkedHashSet<HColumn<ByteBuffer, ByteBuffer>> lastResults;
- /** True if our last load loaded a full page size. */
+ /**
+ * True if our last load loaded a full page size.
+ */
private boolean hasMore = true;
private String currentConnectionType;
public ConnectedIndexScanner( CassandraService cass, String dictionaryType, UUID applicationId, UUID entityId,
- Iterator<String> connectionTypes, ByteBuffer start, boolean reversed, int pageSize ) {
+ Iterator<String> connectionTypes, ByteBuffer start, boolean reversed, int pageSize,
+ boolean skipFirst ) {
Assert.notNull( entityId, "Entity id for row key construction must be specified when searching graph indexes" );
// create our start and end ranges
@@ -74,6 +87,7 @@
this.pageSize = pageSize;
this.dictionaryType = dictionaryType;
this.connectionTypes = connectionTypes;
+ this.skipFirst = skipFirst;
if ( connectionTypes.hasNext() ) {
@@ -106,14 +120,32 @@
return false;
}
+ boolean skipFirst = this.skipFirst && start == scanStart;
+
+ int totalSelectSize = pageSize + 1;
+
+ //we're discarding the first, so increase our total size by 1 since this value will be inclusive in the seek
+ if ( skipFirst ) {
+ totalSelectSize++;
+ }
+
lastResults = new LinkedHashSet<HColumn<ByteBuffer, ByteBuffer>>();
+
+ //cleanup columns for later logic
+ //pointer to the first col we load
+ HColumn<ByteBuffer, ByteBuffer> first = null;
+
+ //pointer to the last column we load
+ HColumn<ByteBuffer, ByteBuffer> last = null;
+
//go through each connection type until we exhaust the result sets
while ( currentConnectionType != null ) {
//only load a delta size to get this next page
- int selectSize = pageSize + 1 - lastResults.size();
+ int selectSize = totalSelectSize - lastResults.size();
+
Object key = key( entityId, dictionaryType, currentConnectionType );
@@ -122,17 +154,24 @@
cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_COMPOSITE_DICTIONARIES, key,
start, null, selectSize, reversed );
+ final int resultSize = results.size();
+
+ if(resultSize > 0){
+
+ last = results.get( resultSize -1 );
+
+ if(first == null ){
+ first = results.get( 0 );
+ }
+ }
+
lastResults.addAll( results );
+
// we loaded a full page, there might be more
- if ( results.size() == selectSize ) {
+ if ( resultSize == selectSize ) {
hasMore = true;
- // set the bytebuffer for the next pass
- start = results.get( results.size() - 1 ).getName();
-
- lastResults.remove( lastResults.size() - 1 );
-
//we've loaded a full page
break;
}
@@ -152,6 +191,16 @@
}
}
+ //remove the first element, we need to skip it
+ if ( skipFirst && first != null) {
+ lastResults.remove( first );
+ }
+
+ if ( hasMore && last != null ) {
+ // set the bytebuffer for the next pass
+ start = last.getName();
+ lastResults.remove( last );
+ }
return lastResults != null && lastResults.size() > 0;
}
@@ -199,7 +248,7 @@
* @see java.util.Iterator#next()
*/
@Override
- @Metered(group = "core", name = "IndexBucketScanner_load")
+ @Metered( group = "core", name = "IndexBucketScanner_load" )
public Set<HColumn<ByteBuffer, ByteBuffer>> next() {
Set<HColumn<ByteBuffer, ByteBuffer>> returnVal = lastResults;
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/IndexBucketScanner.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/IndexBucketScanner.java
index 1a0776a..5c234d9 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/IndexBucketScanner.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/IndexBucketScanner.java
@@ -55,6 +55,7 @@
private final int pageSize;
private final String[] indexPath;
private final IndexType indexType;
+ private final boolean skipFirst;
/** Pointer to our next start read */
private Object start;
@@ -69,9 +70,10 @@
private boolean hasMore = true;
+
public IndexBucketScanner( CassandraService cass, IndexBucketLocator locator, ApplicationCF columnFamily,
UUID applicationId, IndexType indexType, Object keyPrefix, Object start, Object finish,
- boolean reversed, int pageSize, String... indexPath ) {
+ boolean reversed, int pageSize, boolean skipFirst, String... indexPath) {
this.cass = cass;
this.indexBucketLocator = locator;
this.applicationId = applicationId;
@@ -80,7 +82,10 @@
this.start = start;
this.finish = finish;
this.reversed = reversed;
- this.pageSize = pageSize;
+ this.skipFirst = skipFirst;
+
+ //we always add 1 to the page size. This is because we pop the last column for the next page of results
+ this.pageSize = pageSize+1;
this.indexPath = indexPath;
this.indexType = indexType;
this.scanStart = start;
@@ -121,24 +126,40 @@
//if we skip the first we need to set the load to page size +2, since we'll discard the first
//and start paging at the next entity, otherwise we'll just load the page size we need
- int selectSize = pageSize + 1;
+ int selectSize = pageSize;
+
+ //we purposefully use instance equality. If it's a pointer to the same value, we need to increase by 1
+ //since we'll be skipping the first value
+
+ final boolean firstPageSkipFirst = this.skipFirst && start == scanStart;
+
+ if(firstPageSkipFirst){
+ selectSize++;
+ }
TreeSet<HColumn<ByteBuffer, ByteBuffer>> resultsTree = IndexMultiBucketSetLoader
.load( cass, columnFamily, applicationId, cassKeys, start, finish, selectSize, reversed );
+ //remove the first element, it's from a cursor value and we don't want to retain it
+
+
// we loaded a full page, there might be more
if ( resultsTree.size() == selectSize ) {
hasMore = true;
- // set the bytebuffer for the next pass
- start = resultsTree.last().getName();
- resultsTree.remove( resultsTree.last() );
+ // set the bytebuffer for the next pass
+ start = resultsTree.pollLast().getName();
}
else {
hasMore = false;
}
+ //remove the first element since it needs to be skipped AFTER the size check. Otherwise it will fail
+ if ( firstPageSkipFirst ) {
+ resultsTree.pollFirst();
+ }
+
lastResults = resultsTree;
return lastResults != null && lastResults.size() > 0;
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/SearchVisitor.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/SearchVisitor.java
index 665ddd2..02e8386 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/SearchVisitor.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/SearchVisitor.java
@@ -170,8 +170,7 @@
if ( subResults == null ) {
QuerySlice firstFieldSlice = new QuerySlice( slice.getPropertyName(), -1 );
subResults =
- new SliceIterator( slice, secondaryIndexScan( orderByNode, firstFieldSlice ), COLLECTION_PARSER,
- slice.hasCursor() );
+ new SliceIterator( slice, secondaryIndexScan( orderByNode, firstFieldSlice ), COLLECTION_PARSER );
}
orderIterator = new OrderByIterator( slice, orderByNode.getSecondarySorts(), subResults, em,
@@ -190,7 +189,7 @@
scanner = secondaryIndexScan( orderByNode, slice );
}
- SliceIterator joinSlice = new SliceIterator( slice, scanner, COLLECTION_PARSER, slice.hasCursor() );
+ SliceIterator joinSlice = new SliceIterator( slice, scanner, COLLECTION_PARSER);
IntersectionIterator union = new IntersectionIterator( queryProcessor.getPageSizeHint( orderByNode ) );
union.addIterator( joinSlice );
@@ -221,7 +220,7 @@
for ( QuerySlice slice : node.getAllSlices() ) {
IndexScanner scanner = secondaryIndexScan( node, slice );
- intersections.addIterator( new SliceIterator( slice, scanner, COLLECTION_PARSER, slice.hasCursor() ) );
+ intersections.addIterator( new SliceIterator( slice, scanner, COLLECTION_PARSER) );
}
results.push( intersections );
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/IntersectionIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/IntersectionIterator.java
index 4ffad1b..5948cb9 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/IntersectionIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/IntersectionIterator.java
@@ -141,7 +141,9 @@
final Set<ScanColumn> childResults = child.next();
- results.addAll( Sets.intersection( current, childResults ) );
+ final Set<ScanColumn> intersection = Sets.intersection( current, childResults );
+
+ results.addAll( intersection );
}
return results;
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java
index 46807e1..81eacd8 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java
@@ -62,24 +62,36 @@
@Override
public boolean hasNext() {
//if next isn't set, try to advance
- if ( next == null ) {
- doAdvance();
+ if(checkNext()){
+ return true;
}
- boolean results = next != null && next.size() > 0;
- if ( results ) {
- last = next;
- loadCount++;
- }
+ doAdvance();
- return results;
+
+ return checkNext();
+ }
+
+
+ /**
+ * Single source of logic to check if a next is present.
+ * @return
+ */
+ protected boolean checkNext(){
+ return next != null && next.size() > 0;
}
/** Advance to the next page */
protected void doAdvance() {
next = advance();
+
+
+ if ( next != null && next.size() > 0 ) {
+ last = next;
+ loadCount++;
+ }
}
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
index 6cc833d..82f60c2 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
@@ -47,7 +47,6 @@
private final SliceParser parser;
private final IndexScanner scanner;
private final int pageSize;
- private final boolean skipFirst;
/**
* Pointer to the uuid set until it's returned
@@ -74,13 +73,11 @@
* @param scanner The scanner to use to read the cols
* @param slice The slice used in the scanner
* @param parser The parser for the scanner results
- * @param skipFirst True if the first record should be skipped, used with cursors
*/
- public SliceIterator( QuerySlice slice, IndexScanner scanner, SliceParser parser, boolean skipFirst ) {
+ public SliceIterator( QuerySlice slice, IndexScanner scanner, SliceParser parser ) {
this.slice = slice;
this.parser = parser;
this.scanner = scanner;
- this.skipFirst = skipFirst;
this.pageSize = scanner.getPageSize();
this.cols = new LinkedHashMap<UUID, ScanColumn>( this.pageSize );
this.parsedCols = new LinkedHashSet<ScanColumn>( this.pageSize );
@@ -122,13 +119,6 @@
cols.clear();
- /**
- * Skip the first value, it's from the previous cursor
- */
- if ( skipFirst && pagesLoaded == 0 && results.hasNext() ) {
- results.next();
- }
-
parsedCols.clear();
while ( results.hasNext() ) {
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java
index 00e90b8..01daf36 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java
@@ -120,6 +120,16 @@
}
+ @Override
+ public void doReset() {
+ //reset sub iterators if we need to
+ super.doReset();
+
+ list.reset();
+
+ }
+
+
/**
* A Sorted Set with a max size. When a new entry is added, the max is removed. You can mark the next "min" by
* calling the mark method. Values > min are accepted. Values > min and that are over size are discarded
@@ -165,7 +175,7 @@
index = ( index * -1 ) - 1;
- //outside the renage
+ //outside the range
if ( index >= maxSize ) {
return;
}
@@ -224,6 +234,11 @@
public void clear() {
this.list.clear();
}
+
+ public void reset(){
+ clear();
+ this.min = null;
+ }
}
diff --git a/stack/core/src/test/java/org/usergrid/persistence/query/AllInConnectionNoTypeIT.java b/stack/core/src/test/java/org/usergrid/persistence/query/AllInConnectionNoTypeIT.java
index 07d4c46..34f8cfe 100644
--- a/stack/core/src/test/java/org/usergrid/persistence/query/AllInConnectionNoTypeIT.java
+++ b/stack/core/src/test/java/org/usergrid/persistence/query/AllInConnectionNoTypeIT.java
@@ -24,6 +24,7 @@
/** @author tnine */
public class AllInConnectionNoTypeIT extends AbstractIteratingQueryIT {
+
@Test
public void allInConnectionNoType() throws Exception {
allIn( new ConnectionNoTypeHelper(app) );
diff --git a/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java b/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java
index 5f554d3..94fdc40 100644
--- a/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java
+++ b/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java
@@ -25,8 +25,10 @@
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.usergrid.persistence.Entity;
import org.usergrid.persistence.Query;
import org.usergrid.persistence.Results;
+import org.usergrid.persistence.cassandra.QueryProcessor;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -43,7 +45,8 @@
"select * where (field1Or > '00000000' OR field2Or > '00000000') AND fieldDate = '0000-00-00'";
private static final String scanUnion =
"select * where fieldDate = '0000-00-00' AND (field1Or > '00000000' OR field2Or > '00000000') ";
- private static final int PAGE_SIZE = 100;
+
+ private static final int PAGE_SIZE = 300;
@Test
@@ -55,7 +58,7 @@
Set<String> created = performSetup( collectionIoHelper );
- testUnionPaging( collectionIoHelper, unionScan, created );
+// testUnionPaging( collectionIoHelper, unionScan, created );
testUnionPaging( collectionIoHelper, scanUnion, created );
}
@@ -76,7 +79,7 @@
private Set<String> performSetup( final IoHelper io ) throws Exception {
io.doSetup();
- int size = 500;
+ int size = ( int ) ( QueryProcessor.PAGE_SIZE*2.5);
long start = System.currentTimeMillis();
@@ -92,17 +95,16 @@
entity.put( "name", name );
entity.put( "fieldDate", "0000-00-00" );
- String field1;
+ String field1 = String.format( "%08d", i + 1 );
String field2;
//use a value slightly smaller than page size, since we want to simulate
//the cursor issues with union queries
- if ( i < PAGE_SIZE - 10 ) {
- field1 = String.format( "%08d", i + 1 );
+
+ if ( i < size - 10 ) {
field2 = zeros;
}
else {
- field1 = zeros;
field2 = String.format( "%08d", i + 1 );
}
@@ -111,7 +113,9 @@
entity.put( "field1Or", field1 );
entity.put( "field2Or", field2 );
- io.writeEntity( entity );
+ Entity saved = io.writeEntity( entity );
+
+ LOG.info("Writing entity with id '{}'", saved.getUuid());
}
long stop = System.currentTimeMillis();
diff --git a/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java b/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java
index c165caa..ac8c2cb 100644
--- a/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java
+++ b/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java
@@ -49,11 +49,13 @@
// we should get intersection on 1, 3, and 8
InOrderIterator first = new InOrderIterator( 100 );
+ first.add( id9 );
+ first.add( id8 );
first.add( id1 );
first.add( id2 );
first.add( id3 );
- first.add( id8 );
- first.add( id9 );
+
+
InOrderIterator second = new InOrderIterator( 100 );
second.add( id1 );
@@ -89,14 +91,14 @@
// now make sure it's right, only 1, 3 and 8 intersect
assertTrue( union.hasNext() );
+ assertEquals( id8, union.next().getUUID() );
+
+ assertTrue( union.hasNext() );
assertEquals( id1, union.next().getUUID() );
assertTrue( union.hasNext() );
assertEquals( id3, union.next().getUUID() );
- assertTrue( union.hasNext() );
- assertEquals( id8, union.next().getUUID() );
-
assertFalse( union.hasNext() );
}
diff --git a/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/UnionIteratorTest.java b/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/UnionIteratorTest.java
index 8fd9ea8..24851e3 100644
--- a/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/UnionIteratorTest.java
+++ b/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/UnionIteratorTest.java
@@ -365,6 +365,94 @@
}
+ @Test
+ public void resetCorrect() {
+
+ UUID id1 = UUIDUtils.minTimeUUID( 1 );
+ UUID id2 = UUIDUtils.minTimeUUID( 2 );
+ UUID id3 = UUIDUtils.minTimeUUID( 3 );
+ UUID id4 = UUIDUtils.minTimeUUID( 4 );
+ UUID id5 = UUIDUtils.minTimeUUID( 5 );
+ UUID id6 = UUIDUtils.minTimeUUID( 6 );
+ UUID id7 = UUIDUtils.minTimeUUID( 75 );
+
+
+ UnionIterator union = new UnionIterator( 5, 0, null );
+
+ InOrderIterator first = new InOrderIterator( 100 );
+ first.add( id3 );
+ first.add( id6 );
+ first.add( id4 );
+
+
+ InOrderIterator second = new InOrderIterator( 100 );
+ second.add( id7 );
+ second.add( id1 );
+ second.add( id2 );
+ second.add( id5 );
+
+
+ union.addIterator( first );
+ union.addIterator( second );
+
+
+ // now make sure it's right, only 1, 3 and 8 intersect
+ assertTrue( union.hasNext() );
+
+ Set<ScanColumn> ids = union.next();
+
+
+ assertEquals(5, ids.size());
+
+ // now make sure it's right, only 1, 3 and 8 intersect
+ assertTrue( ids.contains( uuidColumn( id1 ) ) );
+ assertTrue( ids.contains( uuidColumn( id2 ) ) );
+ assertTrue( ids.contains( uuidColumn( id3 ) ) );
+ assertTrue( ids.contains( uuidColumn( id4 ) ) );
+ assertTrue( ids.contains( uuidColumn( id5 ) ) );
+
+ ids = union.next();
+
+
+ assertEquals(2, ids.size());
+
+ assertTrue( ids.contains( uuidColumn( id6 ) ) );
+ assertTrue( ids.contains( uuidColumn( id7 ) ) );
+
+ //now try to get the next page
+ ids = union.next();
+ assertNull( ids );
+
+ //now reset and re-test
+ union.reset();
+
+ ids = union.next();
+
+ assertEquals(5, ids.size());
+
+
+ // now make sure it's right, only 1, 3 and 8 intersect
+ assertTrue( ids.contains( uuidColumn( id1 ) ) );
+ assertTrue( ids.contains( uuidColumn( id2 ) ) );
+ assertTrue( ids.contains( uuidColumn( id3 ) ) );
+ assertTrue( ids.contains( uuidColumn( id4 ) ) );
+ assertTrue( ids.contains( uuidColumn( id5 ) ) );
+
+
+ ids = union.next();
+
+ assertEquals(2, ids.size());
+
+ assertTrue( ids.contains( uuidColumn( id6 ) ) );
+ assertTrue( ids.contains( uuidColumn( id7 ) ) );
+
+
+ //now try to get the next page
+ ids = union.next();
+ assertNull( ids );
+ }
+
+
private void reverse( UUID[] array ) {
UUID temp = null;
diff --git a/stack/tools/pom.xml b/stack/tools/pom.xml
index fb5fe95..385303b 100644
--- a/stack/tools/pom.xml
+++ b/stack/tools/pom.xml
@@ -193,6 +193,10 @@
<version>9.1-901.jdbc4</version>
</dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
<!-- Testing and Logging Dependencies -->
<dependency>
diff --git a/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java b/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java
index ba79980..f090a23 100644
--- a/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java
+++ b/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java
@@ -124,9 +124,9 @@
IndexScanner scanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
- indexBucketLocator, applicationId, collectionName );
+ indexBucketLocator, applicationId, collectionName, false );
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser(), false );
+ SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() );
while ( itr.hasNext() ) {
diff --git a/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java
index 367d1da..1b5dff7 100644
--- a/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java
@@ -179,9 +179,9 @@
IndexScanner scanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
- indexBucketLocator, applicationId, collectionName );
+ indexBucketLocator, applicationId, collectionName, false );
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser(), false );
+ SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() );
while ( itr.hasNext() ) {
diff --git a/stack/tools/src/main/java/org/usergrid/tools/bean/ExportOrg.java b/stack/tools/src/main/java/org/usergrid/tools/bean/ExportOrg.java
index bb78c1e..fc5a21b 100644
--- a/stack/tools/src/main/java/org/usergrid/tools/bean/ExportOrg.java
+++ b/stack/tools/src/main/java/org/usergrid/tools/bean/ExportOrg.java
@@ -26,6 +26,7 @@
public class ExportOrg extends OrganizationInfo {
private List<String> adminUserNames;
+ private int passwordHistorySize;
public ExportOrg() {
@@ -55,4 +56,14 @@
public void addAdmin( String username ) {
adminUserNames.add( username );
}
+
+
+ public int getPasswordHistorySize() {
+ return passwordHistorySize;
+ }
+
+
+ public void setPasswordHistorySize( final int passwordHistorySize ) {
+ this.passwordHistorySize = passwordHistorySize;
+ }
}
