http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java index f47afe6..d9ee87e 100644 --- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java @@ -45,6 +45,7 @@ import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; import org.apache.usergrid.persistence.model.field.StringField; +import org.apache.usergrid.persistence.model.util.CollectionUtils; import org.apache.usergrid.persistence.model.util.UUIDGenerator; import org.junit.Before; import org.junit.Test; @@ -205,7 +206,9 @@ public class IndexServiceTest { assertEquals( 1, collectionResults.size() ); - assertEquals( testEntity.getId(), collectionResults.get( 0 ).getId() ); + // with collection versioning, empty versions are included + assertEquals(testEntity.getId().getType(), CollectionUtils.stripEmptyVersion(collectionResults.get(0).getId().getType())); + assertEquals(testEntity.getId().getUuid(), collectionResults.get(0).getId().getUuid()); //query until the connection edge is available @@ -216,7 +219,9 @@ public class IndexServiceTest { assertEquals( 1, connectionResults.size() ); - assertEquals( testEntity.getId(), connectionResults.get( 0 ).getId() ); + // with collection versioning, empty versions are included + assertEquals(testEntity.getId().getType(), CollectionUtils.stripEmptyVersion(collectionResults.get(0).getId().getType())); + assertEquals(testEntity.getId().getUuid(), collectionResults.get(0).getId().getUuid()); } /** @@ -294,8 +299,9 @@ public class IndexServiceTest { assertEquals( 1, collectionResults.size() ); - assertEquals( testEntity.getId(), collectionResults.get( 0 ).getId() ); - + // with collection versioning, empty versions are included + assertEquals(testEntity.getId().getType(), CollectionUtils.stripEmptyVersion(collectionResults.get(0).getId().getType())); + assertEquals(testEntity.getId().getUuid(), collectionResults.get(0).getId().getUuid()); final SearchEdge connectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( connectionSearch ); @@ -306,7 +312,8 @@ public class IndexServiceTest { assertEquals( 1, connectionResults.size() ); - assertEquals( testEntity.getId(), connectionResults.get( 0 ).getId() ); + assertEquals(testEntity.getId().getType(), CollectionUtils.stripEmptyVersion(collectionResults.get(0).getId().getType())); + assertEquals(testEntity.getId().getUuid(), collectionResults.get(0).getId().getUuid()); final SearchEdge lastConnectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( lastSearch ); @@ -318,7 +325,8 @@ public class IndexServiceTest { assertEquals( 1, lastConnectionResults.size() ); - assertEquals( testEntity.getId(), lastConnectionResults.get( 0 ).getId() ); + assertEquals(testEntity.getId().getType(), CollectionUtils.stripEmptyVersion(lastConnectionResults.get(0).getId().getType())); + assertEquals(testEntity.getId().getUuid(), lastConnectionResults.get(0).getId().getUuid()); }
http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java index 281f2af..e9d9e62 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java @@ -163,7 +163,7 @@ public class RebuildIndexTest extends AbstractCoreIT { waitForRebuild( status, reIndexService ); - app.waitForQueueDrainAndRefreshIndex(5000); + app.waitForQueueDrainAndRefreshIndex(10000); // ----------------- test that we can read the catherder collection and not the catshepard @@ -172,7 +172,7 @@ public class RebuildIndexTest extends AbstractCoreIT { } - @Test( timeout = 120000 ) + @Test( timeout = 240000 ) public void rebuildIndex() throws Exception { logger.info( "Started rebuildIndex()" ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java index d22ac65..9b7566a 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java @@ -147,7 +147,7 @@ public class GraphManagerImpl implements GraphManager { public Observable<MarkedEdge> writeEdge( final Edge edge ) { GraphValidation.validateEdge( edge ); - final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, false ); + final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, false, false, false ); final Observable<MarkedEdge> observable = Observable.just( markedEdge ).map( edge1 -> { @@ -178,7 +178,7 @@ public class GraphManagerImpl implements GraphManager { public Observable<MarkedEdge> markEdge( final Edge edge ) { GraphValidation.validateEdge( edge ); - final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, true ); + final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, true, false, false ); final Observable<MarkedEdge> observable = Observable.just( markedEdge ).map( edge1 -> { @@ -269,6 +269,7 @@ public class GraphManagerImpl implements GraphManager { final Observable<MarkedEdge> nodeObservable = Observable.just( inputNode ) .map( node -> nodeSerialization.getMaxVersion( scope, node ) ) + //.doOnNext(maxTimestamp -> logger.info("compactNode maxTimestamp={}", maxTimestamp.toString())) .takeWhile(maxTimestamp -> maxTimestamp.isPresent() ) //map our delete listener .flatMap( timestamp -> nodeDeleteListener.receive( scope, inputNode, startTime ) ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleEdge.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleEdge.java index a89cd96..23ab074 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleEdge.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleEdge.java @@ -23,6 +23,7 @@ package org.apache.usergrid.persistence.graph.impl; import org.apache.usergrid.persistence.graph.Edge; import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation; import org.apache.usergrid.persistence.model.entity.Id; +import org.apache.usergrid.persistence.model.entity.SimpleId; /** @@ -54,6 +55,15 @@ public class SimpleEdge implements Edge { GraphValidation.validateEdge( this ); } + public SimpleEdge( final Edge another, boolean includeEmptyVersion ) { + this( + new SimpleId(another.getSourceNode(), includeEmptyVersion), + another.getType(), + new SimpleId(another.getTargetNode(), includeEmptyVersion), + another.getTimestamp() + ); + } + @Override public Id getSourceNode() { http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java index 9c35e2e..c244ad4 100644 --- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java +++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java @@ -25,6 +25,7 @@ import org.apache.usergrid.persistence.graph.MarkedEdge; import org.apache.usergrid.persistence.model.entity.Id; import com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.usergrid.persistence.model.entity.SimpleId; /** @@ -63,8 +64,22 @@ public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge { } - public SimpleMarkedEdge( final Edge edge, final boolean isDeleted ) { - this( edge.getSourceNode(), edge.getType(), edge.getTargetNode(), edge.getTimestamp(), isDeleted ); + public SimpleMarkedEdge( final Edge edge, final boolean isDeleted, final boolean isSourceNodeDeleted, final boolean isTargetNodeDeleted ) { + this(edge.getSourceNode(), edge.getType(), edge.getTargetNode(), edge.getTimestamp(), + isDeleted, isSourceNodeDeleted, isTargetNodeDeleted); + } + + + public SimpleMarkedEdge( final MarkedEdge another, final boolean includeEmptyVersion ) { + this( + new SimpleId(another.getSourceNode(), includeEmptyVersion), + another.getType(), + new SimpleId(another.getTargetNode(), includeEmptyVersion), + another.getTimestamp(), + another.isDeleted(), + another.isSourceNodeDelete(), + another.isTargetNodeDeleted() + ); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java index b915e4f..707b45f 100644 --- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java +++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java @@ -76,6 +76,12 @@ public class Entity extends EntityObject { this.version = version; } + public Entity(final Entity another, boolean includeEmptyVersion) { + this(new SimpleId(another.getId(), includeEmptyVersion), another.getVersion()); + this.setFieldMap(another.getFieldMap()); + this.setSize(another.getSize()); + } + /** * Generate a new entity with the given type and a new id http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java index 39014ab..6e048ac 100644 --- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java +++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java @@ -44,6 +44,12 @@ public interface Id extends Comparable<Id>, Serializable { */ String getType(); + /** + * Get the unique type for this id, using specified empty version handling + * @return + */ + String getType(boolean includeEmptyVersion); + //Application -> Class "Application" http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java index 6a45558..11bf0e4 100644 --- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java +++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java @@ -29,6 +29,8 @@ import org.apache.usergrid.persistence.model.util.Verify; import com.fasterxml.uuid.UUIDComparator; import com.google.common.base.Preconditions; +import org.apache.usergrid.persistence.model.util.CollectionUtils; + /** @author tnine */ public class SimpleId implements Id, Serializable { @@ -56,6 +58,11 @@ public class SimpleId implements Id, Serializable { this.type = type; } + public SimpleId(final Id another, boolean includeEmptyVersion) { + this.uuid = another.getUuid(); + this.type = another.getType(includeEmptyVersion); + } + /** * Create a new ID. Should only be used for new entities @@ -78,6 +85,18 @@ public class SimpleId implements Id, Serializable { } + @Override + public String getType(boolean includeEmptyVersion) { + String retType; + if (includeEmptyVersion) { + retType = CollectionUtils.addEmptyVersion(type); + } else { + retType = CollectionUtils.stripEmptyVersion(type); + } + return retType; + } + + /** * Do not delete! Needed for Jackson http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/CollectionUtils.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/CollectionUtils.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/CollectionUtils.java new file mode 100644 index 0000000..bff9509 --- /dev/null +++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/CollectionUtils.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.usergrid.persistence.model.util; + +import java.util.HashSet; +import java.util.Set; + +public class CollectionUtils { + public static final String VERSIONED_NAME_SEPARATOR = "~-_~_-~"; + + public static final String COLLECTION_USERS = "users"; + public static final String COLLECTION_GROUPS = "groups"; + public static final String COLLECTION_ASSETS = "assets"; + public static final String COLLECTION_ACTIVITIES = "activities"; + public static final String COLLECTION_EVENTS = "events"; + public static final String COLLECTION_FOLDERS = "folders"; + public static final String COLLECTION_DEVICES = "devices"; + public static final String COLLECTION_NOTIFICATIONS = "notifications"; + public static final String COLLECTION_ROLES = "roles"; + + public static final String COLLECTION_ENTITY_USER = "user"; + public static final String COLLECTION_ENTITY_GROUP = "group"; + public static final String COLLECTION_ENTITY_ASSET = "asset"; + public static final String COLLECTION_ENTITY_ACTIVITY = "activity"; + public static final String COLLECTION_ENTITY_EVENT = "event"; + public static final String COLLECTION_ENTITY_FOLDER = "folder"; + public static final String COLLECTION_ENTITY_DEVICE = "device"; + public static final String COLLECTION_ENTITY_NOTIFICATION = "notification"; + public static final String COLLECTION_ENTITY_ROLE = "role"; + + private static final Set<String> customNames; + + static { + customNames = new HashSet<>(); + customNames.add(COLLECTION_USERS); + customNames.add(COLLECTION_GROUPS); + customNames.add(COLLECTION_ASSETS); + customNames.add(COLLECTION_ACTIVITIES); + customNames.add(COLLECTION_EVENTS); + customNames.add(COLLECTION_FOLDERS); + customNames.add(COLLECTION_DEVICES); + customNames.add(COLLECTION_NOTIFICATIONS); + customNames.add(COLLECTION_ROLES); + + customNames.add(COLLECTION_ENTITY_USER); + customNames.add(COLLECTION_ENTITY_GROUP); + customNames.add(COLLECTION_ENTITY_ASSET); + customNames.add(COLLECTION_ENTITY_ACTIVITY); + customNames.add(COLLECTION_ENTITY_EVENT); + customNames.add(COLLECTION_ENTITY_FOLDER); + customNames.add(COLLECTION_ENTITY_DEVICE); + customNames.add(COLLECTION_ENTITY_NOTIFICATION); + customNames.add(COLLECTION_ENTITY_ROLE); + } + + public static boolean isCustomCollectionOrEntityName(String collectionName) { + return !customNames.contains(collectionName); + } + + public static String stripEmptyVersion(final String name) { + // versioned name with empty version is name followed by separator + if (name.endsWith(VERSIONED_NAME_SEPARATOR)) { + return name.substring(0, name.length() - VERSIONED_NAME_SEPARATOR.length()); + } + return name; + } + + public static String addEmptyVersion(final String name) { + if (!isCustomCollectionOrEntityName(name) || + name.contains(VERSIONED_NAME_SEPARATOR)) { + // not custom collection or already has version + return name; + } + return name + VERSIONED_NAME_SEPARATOR; + } + + public static String handleEmptyVersion(final String name, boolean addEmptyVersion) { + String ret; + if (addEmptyVersion) { + ret = addEmptyVersion(name); + } else { + ret = stripEmptyVersion(name); + } + return ret; + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java index aefceda..b5477f8 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java @@ -33,6 +33,8 @@ import org.apache.usergrid.persistence.model.entity.Id; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId; @@ -43,6 +45,8 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createInd @JsonTypeInfo( use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class" ) public class DeIndexOperation implements BatchOperation { + private static final Logger logger = LoggerFactory.getLogger( DeIndexOperation.class ); + @JsonProperty public String[] indexes; @@ -59,11 +63,13 @@ public class DeIndexOperation implements BatchOperation { UUID version ) { this.indexes = indexes; this.documentId = createIndexDocId( applicationScope, id, version, searchEdge ); + //logger.info("documentId={}", this.documentId); } public DeIndexOperation( String[] indexes, String docId) { this.indexes = indexes; this.documentId = docId; + //logger.info("documentId={}", this.documentId); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java index 1bb97b3..ac2f50a 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java @@ -22,6 +22,7 @@ package org.apache.usergrid.persistence.index.impl; import java.util.Set; import java.util.UUID; +import org.apache.usergrid.persistence.model.entity.SimpleId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,7 +89,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { } //add app id for indexing - container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge, entity,fieldsToIndex)); + container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, + new IndexEdgeImpl(indexEdge, false), + new Entity(entity, false),fieldsToIndex)); return this; } @@ -111,7 +114,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { } - container.addDeIndexRequest(new DeIndexOperation(indexes, applicationScope, searchEdge, id, version)); + container.addDeIndexRequest(new DeIndexOperation(indexes, applicationScope, + new SearchEdgeImpl(searchEdge, false), + new SimpleId(id, false), version)); return this; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java index 5b525cb..409639d 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java @@ -26,6 +26,7 @@ package org.apache.usergrid.persistence.index.impl; import org.apache.usergrid.persistence.index.IndexEdge; import org.apache.usergrid.persistence.model.entity.Id; +import org.apache.usergrid.persistence.model.entity.SimpleId; /** @@ -41,6 +42,15 @@ public class IndexEdgeImpl extends SearchEdgeImpl implements IndexEdge { this.timestamp = timestamp; } + public IndexEdgeImpl( final IndexEdge another, final boolean includeEmptyVersion) { + this( + new SimpleId(another.getNodeId(), includeEmptyVersion), + another.getEdgeName(), + another.getNodeType(), + another.getTimestamp() + ); + } + @Override public long getTimestamp() { http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java index f8560ba..f29e7bc 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java @@ -33,6 +33,8 @@ import org.elasticsearch.client.Client; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -40,6 +42,8 @@ import com.google.common.base.Optional; */ public class IndexOperation implements BatchOperation { + private static final Logger logger = LoggerFactory.getLogger( IndexOperation.class ); + @JsonProperty public String writeAlias; @JsonProperty @@ -61,6 +65,7 @@ public class IndexOperation implements BatchOperation { this.writeAlias = writeAlias; this.data = data; this.documentId = documentId; + //logger.info("documentId={}", documentId); } /** http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java index 5fe5b39..c878ea2 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java @@ -1,4 +1,4 @@ -package org.apache.usergrid.persistence.index.impl;/* +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,12 +17,14 @@ package org.apache.usergrid.persistence.index.impl;/* * under the License. */ +package org.apache.usergrid.persistence.index.impl; import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.usergrid.persistence.core.scope.ApplicationScope; +import org.apache.usergrid.persistence.model.util.CollectionUtils; import org.apache.usergrid.persistence.index.CandidateResult; import org.apache.usergrid.persistence.index.GeoCandidateResult; import org.apache.usergrid.persistence.index.IndexEdge; @@ -69,9 +71,9 @@ public class IndexingUtils { private static final Pattern DOCUMENT_PATTERN = Pattern.compile( DOCUMENT_ID_REGEX ); // These are not allowed in document type names: _ . , | # - public static final String FIELD_SEPERATOR = "."; + public static final String FIELD_SEPARATOR = "."; - public static final String ID_SEPERATOR = ","; + public static final String ID_SEPARATOR = ","; /** @@ -137,13 +139,17 @@ public class IndexingUtils { * * TODO make this format more readable and parsable */ - public static String createContextName( final ApplicationScope applicationScope, final SearchEdge scope ) { + public static String createContextName( final ApplicationScope applicationScope, final SearchEdge searchEdge ) { + SearchEdge strippedSearchEdge = new SearchEdgeImpl( + new SimpleId(searchEdge.getNodeId().getUuid(), CollectionUtils.stripEmptyVersion(searchEdge.getNodeId().getType())), + searchEdge.getEdgeName(), searchEdge.getNodeType() + ); StringBuilder sb = new StringBuilder(); idString( sb, APPID_NAME, applicationScope.getApplication() ); - sb.append( FIELD_SEPERATOR ); - idString( sb, NODEID_NAME, scope.getNodeId() ); - sb.append( FIELD_SEPERATOR ); - appendField( sb, EDGE_NAME, scope.getEdgeName() ); + sb.append(FIELD_SEPARATOR); + idString( sb, NODEID_NAME, strippedSearchEdge.getNodeId() ); + sb.append(FIELD_SEPARATOR); + appendField( sb, EDGE_NAME, strippedSearchEdge.getEdgeName() ); return sb.toString(); } @@ -163,34 +169,41 @@ public class IndexingUtils { public static String createIndexDocId( final ApplicationScope applicationScope, final Id entityId, final UUID version, final SearchEdge searchEdge ) { + // strip empty collection versions to maintain backward compatibility + Id strippedEntityId = new SimpleId(entityId.getUuid(), CollectionUtils.stripEmptyVersion(entityId.getType())); + SearchEdge strippedSearchEdge = new SearchEdgeImpl( + new SimpleId(searchEdge.getNodeId().getUuid(), CollectionUtils.stripEmptyVersion(searchEdge.getNodeId().getType())), + searchEdge.getEdgeName(), searchEdge.getNodeType() + ); + StringBuilder sb = new StringBuilder(); idString( sb, APPID_NAME, applicationScope.getApplication() ); - sb.append( FIELD_SEPERATOR ); - idString( sb, ENTITY_ID_FIELDNAME, entityId ); - sb.append( FIELD_SEPERATOR ); + sb.append(FIELD_SEPARATOR); + idString( sb, ENTITY_ID_FIELDNAME, strippedEntityId ); + sb.append(FIELD_SEPARATOR); appendField( sb, VERSION_NAME, version.toString() ); - sb.append( FIELD_SEPERATOR ); - idString( sb, NODEID_NAME, searchEdge.getNodeId() ); - sb.append( FIELD_SEPERATOR ); - appendField( sb, EDGE_NAME, searchEdge.getEdgeName() ); - sb.append( FIELD_SEPERATOR ); - appendField( sb, NODE_TYPE_NAME, searchEdge.getNodeType().name() ); + sb.append(FIELD_SEPARATOR); + idString( sb, NODEID_NAME, strippedSearchEdge.getNodeId() ); + sb.append(FIELD_SEPARATOR); + appendField( sb, EDGE_NAME, strippedSearchEdge.getEdgeName() ); + sb.append(FIELD_SEPARATOR); + appendField( sb, NODE_TYPE_NAME, strippedSearchEdge.getNodeType().name() ); return sb.toString(); } - public static final String entityId( final Id id ) { + public static String entityId( final Id id ) { return idString( ENTITY_NAME, id ); } - public static final String applicationId( final Id id ) { + public static String applicationId( final Id id ) { return idString( APPID_NAME, id ); } - public static final String nodeId( final Id id ) { + public static String nodeId( final Id id ) { return idString( NODEID_NAME, id ); } @@ -198,7 +211,7 @@ public class IndexingUtils { /** * Construct and Id string with the specified type for the id provided. */ - private static final String idString( final String type, final Id id ) { + private static String idString( final String type, final Id id ) { final StringBuilder stringBuilder = new StringBuilder(); idString( stringBuilder, type, id ); @@ -211,7 +224,7 @@ public class IndexingUtils { * Append the id to the string */ private static final void idString( final StringBuilder builder, final String type, final Id id ) { - builder.append( type ).append( "(" ).append( id.getUuid() ).append( ID_SEPERATOR ) + builder.append( type ).append( "(" ).append( id.getUuid() ).append(ID_SEPARATOR) .append( id.getType().toLowerCase() ).append( ")" ); } @@ -254,7 +267,7 @@ public class IndexingUtils { //Other fields can be parsed using groups. The groups start at value 1, group 0 is the entire match final String entityUUID = matcher.group(3); - final String entityType = matcher.group(4); + final String entityType = CollectionUtils.addEmptyVersion(matcher.group(4)); final String versionUUID = matcher.group(5); @@ -297,8 +310,8 @@ public class IndexingUtils { StringBuilder sb = new StringBuilder(); idString( sb, APPID_NAME, applicationScope.getApplication() ); - sb.append( FIELD_SEPERATOR ); - sb.append( ENTITY_TYPE_NAME).append("(" ).append( type ).append( ")" ); + sb.append(FIELD_SEPARATOR); + sb.append( ENTITY_TYPE_NAME).append("(" ).append( CollectionUtils.stripEmptyVersion(type) ).append( ")" ); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java index 3b62691..7bb2db5 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java @@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.index.impl; import org.apache.usergrid.persistence.index.SearchEdge; import org.apache.usergrid.persistence.model.entity.Id; +import org.apache.usergrid.persistence.model.entity.SimpleId; /** @@ -60,6 +61,14 @@ public class SearchEdgeImpl implements SearchEdge { this.nodeType = nodeType; } + public SearchEdgeImpl( final SearchEdge another, boolean includeEmptyVersion) { + this( + new SimpleId(another.getNodeId(), includeEmptyVersion), + another.getEdgeName(), + another.getNodeType() + ); + } + @Override public Id getNodeId() { http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index ac7d10d..1e75cea 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -32,6 +32,7 @@ import com.google.common.base.Optional; import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.index.*; import org.apache.usergrid.persistence.model.field.*; +import org.apache.usergrid.persistence.model.util.CollectionUtils; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -170,24 +171,29 @@ public class EntityIndexTest extends BaseIT { final CandidateResult candidate1 = candidateResults.get(0); //check the id and version - assertEquals( entity1.getId(), candidate1.getId() ); + // with collection versioning, empty versions are included in results + assertEquals(entity1.getId().getType(), CollectionUtils.stripEmptyVersion(candidate1.getId().getType())); + assertEquals(entity1.getId().getUuid(), candidate1.getId().getUuid()); assertEquals(entity1.getVersion(), candidate1.getVersion()); final CandidateResult candidate2 = candidateResults.get(1); //check the id and version - assertEquals( entity2.getId(), candidate2.getId() ); + assertEquals(entity2.getId().getType(), CollectionUtils.stripEmptyVersion(candidate2.getId().getType())); + assertEquals(entity2.getId().getUuid(), candidate2.getId().getUuid()); assertEquals( entity2.getVersion(), candidate2.getVersion() ); //make sure we can query uuids out as strings and not wrapped candidateResults = entityIndex.search( indexEdge, searchTypes, "select * where testuuid = '"+uuid+"'", 100, 0, false ); - assertEquals(entity1.getId(),candidateResults.get(0).getId()); + assertEquals(entity1.getId().getType(), CollectionUtils.stripEmptyVersion(candidateResults.get(0).getId().getType())); + assertEquals(entity1.getId().getUuid(), candidateResults.get(0).getId().getUuid()); candidateResults = entityIndex.search( indexEdge, searchTypes, "select * where testuuid = "+uuid, 100, 0, false); - assertEquals(entity1.getId(),candidateResults.get(0).getId()); + assertEquals(entity1.getId().getType(), CollectionUtils.stripEmptyVersion(candidateResults.get(0).getId().getType())); + assertEquals(entity1.getId().getUuid(), candidateResults.get(0).getId().getUuid()); } @@ -519,7 +525,8 @@ public class EntityIndexTest extends BaseIT { final String query = "where username = 'edanuff'"; CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false); - assertEquals( user.getId(), r.get( 0 ).getId()); + assertEquals(user.getId().getType(), CollectionUtils.stripEmptyVersion(r.get(0).getId().getType())); + assertEquals(user.getId().getUuid(), r.get(0).getId().getUuid()); batch.deindex( indexSCope, user.getId(), user.getVersion() ); indexProducer.put(batch.build()).subscribe();; @@ -734,7 +741,10 @@ public class EntityIndexTest extends BaseIT { final CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes(entityId.getType()), query, 10, 0, false); - assertEquals(user.getId(), r.get(0).getId()); + + // with collection versioning, empty versions are included in results + assertEquals(user.getId().getType(), CollectionUtils.stripEmptyVersion(r.get(0).getId().getType())); + assertEquals(user.getId().getUuid(), r.get(0).getId().getUuid()); } @@ -774,7 +784,9 @@ public class EntityIndexTest extends BaseIT { final CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), query, 10, 0, false); - assertEquals(user.getId(), r.get(0).getId()); + // with collection versioning, empty versions are included in results + assertEquals(user.getId().getType(), CollectionUtils.stripEmptyVersion(r.get(0).getId().getType())); + assertEquals(user.getId().getUuid(), r.get(0).getId().getUuid()); //shouldn't match final String queryNoWildCard = "where string = 'I am'"; @@ -833,8 +845,11 @@ public class EntityIndexTest extends BaseIT { entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), ascQuery, 10 , 0, false); - assertEquals( first.getId(), ascResults.get( 0).getId() ); - assertEquals( second.getId(), ascResults.get( 1 ).getId() ); + // with collection versioning, empty versions are included in results + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(ascResults.get(0).getId().getType())); + assertEquals(first.getId().getUuid(), ascResults.get(0).getId().getUuid()); + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(ascResults.get(1).getId().getType())); + assertEquals(second.getId().getUuid(), ascResults.get(1).getId().getUuid()); //search in reversed @@ -844,8 +859,11 @@ public class EntityIndexTest extends BaseIT { entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), descQuery, 10 , 0, false); - assertEquals( second.getId(), descResults.get( 0).getId() ); - assertEquals( first.getId(), descResults.get( 1 ).getId() ); + // with collection versioning, empty versions are included when parsing doc IDs + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(descResults.get(0).getId().getType())); + assertEquals(second.getId().getUuid(), descResults.get(0).getId().getUuid()); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(descResults.get(1).getId().getType())); + assertEquals(first.getId().getUuid(), descResults.get(1).getId().getUuid()); } @@ -899,7 +917,8 @@ public class EntityIndexTest extends BaseIT { assertEquals(1, singleResults.size()); - assertEquals(first.getId(), singleResults.get(0).getId()); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(singleResults.get(0).getId().getType())); + assertEquals(first.getId().getUuid(), singleResults.get(0).getId().getUuid()); //search in reversed @@ -910,8 +929,11 @@ public class EntityIndexTest extends BaseIT { assertEquals( 2, singleKeywordUnion.size() ); - assertEquals( second.getId(), singleKeywordUnion.get( 0).getId() ); - assertEquals( first.getId(), singleKeywordUnion.get( 1 ).getId() ); + // with collection versioning, empty versions are included when parsing doc IDs + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(singleKeywordUnion.get(0).getId().getType())); + assertEquals(second.getId().getUuid(), singleKeywordUnion.get(0).getId().getUuid()); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(singleKeywordUnion.get(1).getId().getType())); + assertEquals(first.getId().getUuid(), singleKeywordUnion.get(1).getId().getUuid()); final String twoKeywordMatches = "string contains 'alpha' OR string contains 'long'"; @@ -921,8 +943,10 @@ public class EntityIndexTest extends BaseIT { assertEquals( 2, towMatchResults.size() ); - assertEquals(second.getId(), towMatchResults.get( 0).getId() ); - assertEquals(first.getId(), towMatchResults.get( 1 ).getId() ); + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(towMatchResults.get(0).getId().getType())); + assertEquals(second.getId().getUuid(), towMatchResults.get(0).getId().getUuid()); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(towMatchResults.get(1).getId().getType())); + assertEquals(first.getId().getUuid(), towMatchResults.get(1).getId().getUuid()); } @@ -980,7 +1004,8 @@ public class EntityIndexTest extends BaseIT { assertEquals( 1, notFirstResults.size() ); - assertEquals(second.getId(), notFirstResults.get( 0 ).getId() ); + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notFirstResults.get(0).getId().getType())); + assertEquals(second.getId().getUuid(), notFirstResults.get(0).getId().getUuid()); //search in reversed @@ -991,7 +1016,8 @@ public class EntityIndexTest extends BaseIT { assertEquals( 1, notSecondUnion.size() ); - assertEquals( first.getId(), notSecondUnion.get( 0 ).getId() ); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notSecondUnion.get(0).getId().getType())); + assertEquals(first.getId().getUuid(), notSecondUnion.get(0).getId().getUuid()); final String notBothReturn = "NOT int = 3"; @@ -1001,8 +1027,10 @@ public class EntityIndexTest extends BaseIT { assertEquals( 2, notBothReturnResults.size() ); - assertEquals( second.getId(), notBothReturnResults.get( 0).getId() ); - assertEquals( first.getId(), notBothReturnResults.get( 1 ).getId() ); + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notBothReturnResults.get(0).getId().getType())); + assertEquals(second.getId().getUuid(), notBothReturnResults.get(0).getId().getUuid()); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notBothReturnResults.get(1).getId().getType())); + assertEquals(first.getId().getUuid(), notBothReturnResults.get(1).getId().getUuid()); final String notFilterBoth = "(NOT int = 1) AND (NOT int = 2) "; @@ -1020,8 +1048,10 @@ public class EntityIndexTest extends BaseIT { assertEquals( 2, noMatchesAndResults.size() ); - assertEquals( second.getId(), noMatchesAndResults.get( 0).getId() ); - assertEquals( first.getId(), noMatchesAndResults.get( 1 ).getId() ); + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(noMatchesAndResults.get(0).getId().getType())); + assertEquals(second.getId().getUuid(), noMatchesAndResults.get(0).getId().getUuid()); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(noMatchesAndResults.get(1).getId().getType())); + assertEquals(first.getId().getUuid(), noMatchesAndResults.get(1).getId().getUuid()); final String noMatchesOr = "(NOT int = 3) AND (NOT int = 4)"; @@ -1031,8 +1061,10 @@ public class EntityIndexTest extends BaseIT { assertEquals( 2, noMatchesOrResults.size() ); - assertEquals( second.getId(), noMatchesOrResults.get( 0).getId() ); - assertEquals( first.getId(), noMatchesOrResults.get( 1 ).getId() ); + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(noMatchesOrResults.get(0).getId().getType())); + assertEquals(second.getId().getUuid(), noMatchesOrResults.get(0).getId().getUuid()); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(noMatchesOrResults.get(1).getId().getType())); + assertEquals(first.getId().getUuid(), noMatchesOrResults.get(1).getId().getUuid()); } @@ -1090,7 +1122,9 @@ public class EntityIndexTest extends BaseIT { assertEquals( 1, notFirstResults.size() ); - assertEquals(second.getId(), notFirstResults.get( 0 ).getId() ); + // with collection versioning, empty versions are included + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notFirstResults.get(0).getId().getType())); + assertEquals(second.getId().getUuid(), notFirstResults.get(0).getId().getUuid()); final String notFirstWildCard = "NOT string = 'I ate*'"; @@ -1100,7 +1134,8 @@ public class EntityIndexTest extends BaseIT { assertEquals( 1, notFirstWildCardResults.size() ); - assertEquals(second.getId(), notFirstWildCardResults.get( 0 ).getId() ); + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notFirstWildCardResults.get(0).getId().getType())); + assertEquals(second.getId().getUuid(), notFirstWildCardResults.get(0).getId().getUuid()); final String notFirstContains = "NOT string contains 'sammich'"; @@ -1110,7 +1145,8 @@ public class EntityIndexTest extends BaseIT { assertEquals( 1, notFirstContainsResults.size() ); - assertEquals(second.getId(), notFirstContainsResults.get( 0 ).getId() ); + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notFirstContainsResults.get(0).getId().getType())); + assertEquals(second.getId().getUuid(), notFirstContainsResults.get(0).getId().getUuid()); //search in reversed @@ -1121,7 +1157,8 @@ public class EntityIndexTest extends BaseIT { assertEquals( 1, notSecondUnion.size() ); - assertEquals( first.getId(), notSecondUnion.get( 0 ).getId() ); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notSecondUnion.get(0).getId().getType())); + assertEquals(first.getId().getUuid(), notSecondUnion.get(0).getId().getUuid()); final String notSecondWildcard = "NOT string = 'I drank*'"; @@ -1131,7 +1168,8 @@ public class EntityIndexTest extends BaseIT { assertEquals( 1, notSecondWildcardUnion.size() ); - assertEquals( first.getId(), notSecondWildcardUnion.get( 0 ).getId() ); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notSecondWildcardUnion.get(0).getId().getType())); + assertEquals(first.getId().getUuid(), notSecondWildcardUnion.get(0).getId().getUuid()); final String notSecondContains = "NOT string contains 'beer'"; @@ -1141,7 +1179,8 @@ public class EntityIndexTest extends BaseIT { assertEquals( 1, notSecondContainsUnion.size() ); - assertEquals( first.getId(), notSecondContainsUnion.get( 0 ).getId() ); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notSecondContainsUnion.get(0).getId().getType())); + assertEquals(first.getId().getUuid(), notSecondContainsUnion.get(0).getId().getUuid()); final String notBothReturn = "NOT string = 'I'm a foodie'"; @@ -1151,8 +1190,10 @@ public class EntityIndexTest extends BaseIT { assertEquals( 2, notBothReturnResults.size() ); - assertEquals( second.getId(), notBothReturnResults.get( 0).getId() ); - assertEquals( first.getId(), notBothReturnResults.get( 1 ).getId() ); + assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notBothReturnResults.get(0).getId().getType())); + assertEquals(second.getId().getUuid(), notBothReturnResults.get(0).getId().getUuid()); + assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notBothReturnResults.get(1).getId().getType())); + assertEquals(first.getId().getUuid(), notBothReturnResults.get(1).getId().getUuid()); final String notFilterBoth = "(NOT string = 'I ate a sammich') AND (NOT string = 'I drank a beer') "; @@ -1299,7 +1340,8 @@ public class EntityIndexTest extends BaseIT { final CandidateResult candidate1 = candidateResults.get(0); //check the id and version - assertEquals( entity1.getId(), candidate1.getId() ); + assertEquals(entity1.getId().getType(), CollectionUtils.stripEmptyVersion(candidate1.getId().getType())); + assertEquals(entity1.getId().getUuid(), candidate1.getId().getUuid()); assertEquals(entity1.getVersion(), candidate1.getVersion()); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java index ba33030..91f41b6 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java @@ -28,6 +28,7 @@ import java.util.*; import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.index.*; import org.apache.usergrid.persistence.model.entity.SimpleId; +import org.apache.usergrid.persistence.model.util.CollectionUtils; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -101,7 +102,7 @@ public class GeoPagingTest extends BaseIT { /** * Test that geo-query returns co-located entities in expected order. - */ + nde*/ @Test public void groupQueriesWithDistanceOrderedResults() throws Exception { @@ -150,7 +151,9 @@ public class GeoPagingTest extends BaseIT { final Entity expected = cats[consistent]; - assertEquals(expected.getId(), candidate.getId()); + // with collection versioning, empty versions are included + assertEquals(expected.getId().getType(), CollectionUtils.stripEmptyVersion(candidate.getId().getType())); + assertEquals(expected.getId().getUuid(), candidate.getId().getUuid()); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java index d93f8a3..af94864 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java @@ -22,6 +22,7 @@ package org.apache.usergrid.persistence.index.impl; import java.util.UUID; +import org.apache.usergrid.persistence.model.util.CollectionUtils; import org.junit.Test; import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; @@ -87,7 +88,9 @@ public class IndexingUtilsTest { final CandidateResult parsedId = parseIndexDocId( output ); assertEquals(version, parsedId.getVersion()); - assertEquals(id, parsedId.getId()); + // with collection versioning, empty versions are included when parsing doc IDs + assertEquals(id.getType(), CollectionUtils.stripEmptyVersion(parsedId.getId().getType())); + assertEquals(id.getUuid(), parsedId.getId().getUuid()); } @@ -119,7 +122,9 @@ public class IndexingUtilsTest { final CandidateResult parsedId = parseIndexDocId( output ); assertEquals(version, parsedId.getVersion()); - assertEquals(id, parsedId.getId()); + // with collection versioning, empty versions are included when parsing doc IDs + assertEquals(id.getType(), CollectionUtils.stripEmptyVersion(parsedId.getId().getType())); + assertEquals(id.getUuid(), parsedId.getId().getUuid()); final UUID appId = parseAppIdFromIndexDocId(output); assertEquals(appId,applicationScope.getApplication().getUuid()); @@ -154,7 +159,9 @@ public class IndexingUtilsTest { final CandidateResult parsedId = parseIndexDocId( output ); assertEquals(version, parsedId.getVersion()); - assertEquals(id, parsedId.getId()); + // with collection versioning, empty versions are included when parsing doc IDs + assertEquals(id.getType(), CollectionUtils.stripEmptyVersion(parsedId.getId().getType())); + assertEquals(id.getUuid(), parsedId.getId().getUuid()); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queue/src/test/resources/qakka.properties ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queue/src/test/resources/qakka.properties b/stack/corepersistence/queue/src/test/resources/qakka.properties index d77e7e8..f78be41 100644 --- a/stack/corepersistence/queue/src/test/resources/qakka.properties +++ b/stack/corepersistence/queue/src/test/resources/qakka.properties @@ -18,7 +18,7 @@ # Properties for JUnit tests -queue.standalone=true +queue.standalone=false usergrid.cluster_name=Test Cluster http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java index 86b3216..f2e428d 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java @@ -31,7 +31,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.PathSegment; import javax.ws.rs.core.UriInfo; -import org.apache.usergrid.persistence.entities.Application; +import org.apache.usergrid.persistence.model.util.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; @@ -89,7 +89,7 @@ public class CollectionResource extends ServiceResource { logger.trace( "CollectionResource.executeClearCollection" ); } - if (!Application.isCustomCollectionName(itemName.getPath())) { + if (!CollectionUtils.isCustomCollectionOrEntityName(itemName.getPath())) { throw new IllegalArgumentException( "Cannot clear built-in collections (" + itemName + ")." ); @@ -139,7 +139,7 @@ public class CollectionResource extends ServiceResource { logger.trace( "CollectionResource.executeGetCollectionVersion" ); } - if (!Application.isCustomCollectionName(itemName.getPath())) { + if (!CollectionUtils.isCustomCollectionOrEntityName(itemName.getPath())) { throw new IllegalArgumentException( "Built-in collections are not versioned." ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java index 14ed54f..306231f 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.jaxrs.json.annotation.JSONP; import com.google.cloud.storage.StorageException; import org.apache.commons.lang.StringUtils; -import org.apache.usergrid.corepersistence.index.CollectionVersionUtil; import org.apache.usergrid.corepersistence.index.VersionedCollectionName; import org.apache.usergrid.management.OrganizationConfig; import org.apache.usergrid.management.OrganizationConfigProps; @@ -40,6 +39,7 @@ import org.apache.usergrid.services.*; import org.apache.usergrid.services.assets.BinaryStoreFactory; import org.apache.usergrid.services.assets.data.*; import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException; +import org.apache.usergrid.corepersistence.index.CollectionVersionUtils; import org.apache.usergrid.utils.JsonUtils; import org.glassfish.jersey.media.multipart.BodyPart; import org.glassfish.jersey.media.multipart.BodyPartEntity; @@ -47,7 +47,6 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataMultiPart; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.BeanInfoFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -431,7 +430,7 @@ public class ServiceResource extends AbstractContextResource { for (int i = 0; i < r.getEntities().size(); i++) { Entity e = r.getEntity(i); String oldType = e.getType(); - VersionedCollectionName v = CollectionVersionUtil.parseVersionedName(oldType); + VersionedCollectionName v = CollectionVersionUtils.parseVersionedName(oldType); if (v.hasVersion()) { e.setType(v.getCollectionName()); r.setEntity(i, e); http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionClearTooSoonExceptionMapper.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionClearTooSoonExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionClearTooSoonExceptionMapper.java new file mode 100644 index 0000000..bc49d2a --- /dev/null +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionClearTooSoonExceptionMapper.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.rest.exceptions; + + +import org.apache.usergrid.corepersistence.asyncevents.CollectionClearTooSoonException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.Provider; + +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; + + +@Provider +public class CollectionClearTooSoonExceptionMapper extends AbstractExceptionMapper<CollectionClearTooSoonException> { + + private static final Logger logger = LoggerFactory.getLogger(CollectionClearTooSoonExceptionMapper.class); + + @Override + public Response toResponse( CollectionClearTooSoonException e ) { + + if(logger.isTraceEnabled()) { + logger.trace("Tried to clear collection too soon after previous clear", e.getMessage()); + } + + return toResponse( BAD_REQUEST, e ); + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionDeleteTooSoonExceptionMapper.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionDeleteTooSoonExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionDeleteTooSoonExceptionMapper.java deleted file mode 100644 index 3b247df..0000000 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionDeleteTooSoonExceptionMapper.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.usergrid.rest.exceptions; - - -import org.apache.usergrid.corepersistence.asyncevents.CollectionDeleteTooSoonException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.ws.rs.core.Response; -import javax.ws.rs.ext.Provider; - -import static javax.ws.rs.core.Response.Status.BAD_REQUEST; - - -@Provider -public class CollectionDeleteTooSoonExceptionMapper extends AbstractExceptionMapper<CollectionDeleteTooSoonException> { - - private static final Logger logger = LoggerFactory.getLogger(CollectionDeleteTooSoonExceptionMapper.class); - - @Override - public Response toResponse( CollectionDeleteTooSoonException e ) { - - if(logger.isTraceEnabled()) { - logger.trace("Tried to delete collection too soon after previous deletion", e.getMessage()); - } - - return toResponse( BAD_REQUEST, e ); - } -} http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java index be60177..3752b08 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java @@ -354,6 +354,9 @@ public class IndexResource extends AbstractContextResource { response.setProperty( "status", status.getStatus() ); response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() ); response.setProperty( "numberQueued", status.getNumberProcessed() ); + if (request.getUpdateTimestamp().isPresent() && request.getUpdateTimestamp().get() > 0L) { + response.setProperty("updatedSince", request.getUpdateTimestamp()); + } response.setSuccess(); return response; http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionClearTest.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionClearTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionClearTest.java index e40c193..b25a717 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionClearTest.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionClearTest.java @@ -44,7 +44,7 @@ public class CollectionClearTest extends AbstractRestIT { * @throws Exception */ @Test - public void collectionDelete() throws Exception { + public void collectionClear() throws Exception { String collectionName = "children"; int numEntities = 10; @@ -54,47 +54,98 @@ public class CollectionClearTest extends AbstractRestIT { String namePrefixAfterClear = "abc"; // verify collection version is empty - ApiResponse tempResponse = this.app().collection(collectionName).collection("_version").get().getResponse(); - LinkedHashMap dataMap = (LinkedHashMap)tempResponse.getData(); - assertEquals("", dataMap.get("version")); - assertEquals(collectionName, dataMap.get("collectionName")); + String collectionVersion = getCollectionVersion(collectionName); + assertEquals("", collectionVersion); - createEntities( collectionName, namePrefix, numEntities ); + createEntities( collectionName, namePrefix, 1, numEntities ); // retrieve entities, provide 1 more than num entities - QueryParameters parms = new QueryParameters().setLimit( numEntities + 1 ).setQuery("order by created asc"); - List<Entity> entities = retrieveEntities(collectionName, namePrefix, parms, numEntities, false); + QueryParameters parms = new QueryParameters().setLimit( numEntities + 1 ); + List<Entity> entities = retrieveEntities(collectionName, namePrefix, parms, 1, numEntities, true); assertEquals(numEntities, entities.size()); // clear the collection - Map<String, Object> payload = new HashMap<String, Object>(); + Map<String, Object> payload = new HashMap<>(); parms = new QueryParameters().setKeyValue("confirm_collection_name", collectionName); - tempResponse = this.app().collection(collectionName).collection("_clear").post(true, payload, parms); + ApiResponse clearResponse = this.app().collection(collectionName).collection("_clear").post(true, payload, parms); // verify collection version has changed - tempResponse = this.app().collection(collectionName).collection("_version").get().getResponse(); - dataMap = (LinkedHashMap)tempResponse.getData(); - String newVersion = (String)dataMap.get("version"); + String newVersion = getCollectionVersion(collectionName); assertNotEquals("", newVersion); - assertEquals(collectionName, dataMap.get("collectionName")); // validate that 0 entities left - List<Entity> entitiesAfterClear = retrieveEntities(collectionName, namePrefix, parms, 0, true); + List<Entity> entitiesAfterClear = retrieveEntities(collectionName, namePrefix, parms, 1, 0, true); assertEquals(0, entitiesAfterClear.size()); // insert more entities using same collectionName - createEntities( collectionName, namePrefixAfterClear, numEntitiesAfterClear ); + createEntities( collectionName, namePrefixAfterClear, 1, numEntitiesAfterClear ); // validate correct number of entities - parms = new QueryParameters().setLimit( numEntitiesAfterClear + 1 ).setQuery("order by created asc"); - List<Entity> newEntities = retrieveEntities(collectionName, namePrefixAfterClear, parms, numEntitiesAfterClear, false); + parms = new QueryParameters().setLimit( numEntitiesAfterClear + 1 ); + List<Entity> newEntities = retrieveEntities(collectionName, namePrefixAfterClear, parms, 1, numEntitiesAfterClear, true); assertEquals(numEntitiesAfterClear, newEntities.size()); // verify collection version has not changed - tempResponse = this.app().collection(collectionName).collection("_version").get().getResponse(); - dataMap = (LinkedHashMap)tempResponse.getData(); - assertEquals(newVersion, dataMap.get("version")); + String lastVersion = getCollectionVersion(collectionName); + assertEquals(newVersion, lastVersion); + } + + + /** + * Tests that old collection entities are deleted. + * @throws Exception + */ + @Test + public void collectionMultipleClear() throws Exception { + String collectionName = "dogs"; + int numEntities = 2000; + String namePrefix = "dog"; + int numDeleteCycles = 3; + int startingEntityNum = 1; + + // should start out as unversioned + String currentVersion = getCollectionVersion(collectionName); + assertEquals("", currentVersion); + + for (int cycle = 1; cycle <= numDeleteCycles; cycle++) { + logger.info("Creating entities {} - {} for cycle {}", startingEntityNum, lastEntityNum(startingEntityNum, numEntities), cycle); + createEntities( collectionName, namePrefix, startingEntityNum, numEntities ); + + // retrieve entities, provide 1 more than num entities + logger.info("Retrieving entities {} - {} for cycle {}", startingEntityNum, lastEntityNum(startingEntityNum, numEntities), cycle); + QueryParameters parms = new QueryParameters().setLimit( numEntities + 1 ); + List<Entity> entities = retrieveEntities(collectionName, namePrefix, parms, startingEntityNum, numEntities, true); + assertEquals(numEntities, entities.size()); + + // clear collection + logger.info("Clearing collection for cycle {}", cycle); + String newVersion = clearCollection(collectionName); + logger.info("Collection version is {} for cycle {}", newVersion, cycle); + assertNotEquals(currentVersion, newVersion); + + // validate that 0 entities left + List<Entity> entitiesAfterClear = retrieveEntities(collectionName, namePrefix, parms, 1, 0, true); + assertEquals(0, entitiesAfterClear.size()); + + currentVersion = newVersion; + startingEntityNum = startingEntityNum + numEntities; + } + + } + + private int lastEntityNum(int startingEntityNum, int numEntities) { + return startingEntityNum + numEntities - 1; + } + + + /** + * Get collection version + */ + private String getCollectionVersion(String collectionName) { + ApiResponse tempResponse = this.app().collection(collectionName).collection("_version").get().getResponse(); + LinkedHashMap dataMap = (LinkedHashMap)tempResponse.getData(); assertEquals(collectionName, dataMap.get("collectionName")); + return (String)dataMap.get("version"); } @@ -104,10 +155,10 @@ public class CollectionClearTest extends AbstractRestIT { * @param collectionName * @param numOfEntities */ - public List<Entity> createEntities(String collectionName, String namePrefix, int numOfEntities ){ + public List<Entity> createEntities(String collectionName, String namePrefix, int firstEntity, int numOfEntities ){ List<Entity> entities = new LinkedList<>( ); - for ( int i = 1; i <= numOfEntities; i++ ) { + for ( int i = firstEntity; i <= lastEntityNum(firstEntity, numOfEntities); i++ ) { Map<String, Object> entityPayload = new HashMap<String, Object>(); entityPayload.put( "name", namePrefix + String.valueOf( i ) ); entityPayload.put( "num", i ); @@ -117,12 +168,8 @@ public class CollectionClearTest extends AbstractRestIT { entities.add( entity ); this.app().collection( collectionName ).post( entity ); - - if ( i % 100 == 0){ - logger.info("created {} entities", i); - } } - logger.info("created {} total entities", numOfEntities); + logger.info("created {} entities", numOfEntities); this.waitForQueueDrainAndRefreshIndex(); @@ -135,15 +182,15 @@ public class CollectionClearTest extends AbstractRestIT { * @param parms * @param numOfEntities */ - public List<Entity> retrieveEntities(String collectionName, String namePrefix, QueryParameters parms, int numOfEntities, boolean reverseOrder){ + public List<Entity> retrieveEntities(String collectionName, String namePrefix, QueryParameters parms, int firstEntity, int numOfEntities, boolean reverseOrder){ List<Entity> entities = new LinkedList<>( ); Collection testCollection = this.app().collection( collectionName ).get(parms, true); int entityNum; if (reverseOrder) { - entityNum = numOfEntities; + entityNum = lastEntityNum(firstEntity, numOfEntities); } else { - entityNum = 1; + entityNum = firstEntity; } while (testCollection.getCursor() != null) { while (testCollection.hasNext()) { @@ -176,4 +223,16 @@ public class CollectionClearTest extends AbstractRestIT { return entities; } + private String clearCollection(String collectionName) { + // clear the collection + Map<String, Object> payload = new HashMap<>(); + QueryParameters parms = new QueryParameters().setKeyValue("confirm_collection_name", collectionName); + ApiResponse clearResponse = this.app().collection(collectionName).collection("_clear").post(true, payload, parms); + + // verify collection version has changed + String newVersion = getCollectionVersion(collectionName); + + return newVersion; + } + } http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java index dee78f9..10ac63a 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java @@ -17,6 +17,7 @@ package org.apache.usergrid.services; +import org.apache.usergrid.corepersistence.index.CollectionVersionUtils; import org.apache.usergrid.persistence.*; import org.apache.usergrid.persistence.Query.Level; import org.apache.usergrid.persistence.index.query.Identifier; @@ -32,13 +33,16 @@ import rx.Observable; import rx.schedulers.Schedulers; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; +import static org.apache.usergrid.services.ServiceManager.MAP_VERSIONED_COLLECTION_NAME_KEY; import static org.apache.usergrid.services.ServiceParameter.filter; import static org.apache.usergrid.services.ServiceParameter.firstParameterIsName; import static org.apache.usergrid.utils.ClassUtils.cast; import static org.apache.usergrid.utils.InflectionUtils.pluralize; +import static org.apache.usergrid.utils.InflectionUtils.singularize; import static org.apache.usergrid.utils.ListUtils.dequeue; import static org.apache.usergrid.utils.ListUtils.initCopy; @@ -90,6 +94,9 @@ public class AbstractConnectionsService extends AbstractService { cType = dequeue( parameters ).getName(); } if ( cType != null ) { + // this is not a versionable collection. If there is a version here, it is because the connection name matches + // a versioned collection. Remove the version. + cType = CollectionVersionUtils.getBaseCollectionName(cType); collectionName = cType; } @@ -121,6 +128,9 @@ public class AbstractConnectionsService extends AbstractService { } else { eType = Schema.normalizeEntityType( s ); + Map<String,String> collectionInfo = sm.getVersionedCollectionInfo(s, eType); + eType = collectionInfo.get(MAP_VERSIONED_COLLECTION_NAME_KEY); + //logger.info("connection service collection eType:{}", eType); first_parameter = dequeue( parameters ); if ( first_parameter instanceof QueryParameter ) { query = first_parameter.getQuery(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java index 711a86c..1f5aeb4 100644 --- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java +++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java @@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit; import org.apache.usergrid.corepersistence.index.CollectionScopeImpl; import org.apache.usergrid.corepersistence.index.CollectionVersionManager; import org.apache.usergrid.corepersistence.index.CollectionVersionManagerFactory; -import org.apache.usergrid.corepersistence.index.CollectionVersionUtil; +import org.apache.usergrid.corepersistence.index.CollectionVersionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; @@ -49,6 +49,7 @@ import com.google.common.cache.LoadingCache; import static org.apache.usergrid.persistence.SimpleEntityRef.ref; import static org.apache.usergrid.utils.InflectionUtils.pluralize; +import static org.apache.usergrid.utils.InflectionUtils.singularize; public class ServiceManager { @@ -225,6 +226,57 @@ public class ServiceManager { } + public static String MAP_VERSIONED_COLLECTION_NAME_KEY = "versionedName"; + public static String MAP_UNVERSIONED_COLLECTION_NAME_KEY = "unversionedName"; + public static String MAP_VERSIONED_ITEM_TYPE_KEY = "versionedType"; + public static String MAP_UNVERSIONED_ITEM_TYPE_KEY = "unversionedType"; + public static String MAP_COLLECTION_VERSION_KEY = "version"; + + /** + * Returns versioned collection info in Map ( + * @param collectionName + * @param itemType + * @return + */ + public Map<String,String> getVersionedCollectionInfo(String collectionName, String itemType) { + Map<String,String> collectionInfo = new HashMap<>(); + String versionedCollectionName = collectionName; + String unversionedCollectionName = CollectionVersionUtils.getBaseCollectionName(collectionName); + String versionedItemType = itemType; + // this works for item type too + String unversionedItemType = CollectionVersionUtils.getBaseCollectionName(itemType); + String collectionVersion = ""; + if (collectionName.equals(unversionedCollectionName)) { + // no version passed in + CollectionVersionManager collectionVersionManager = cvmf.getInstance(new CollectionScopeImpl(applicationId, unversionedCollectionName)); + // always bypass collection version cache for now + collectionVersion = collectionVersionManager.getCollectionVersion(true); + + if (collectionVersion != "") { + if (logger.isTraceEnabled()) { + logger.trace("getVersionedCollectionName: currentCollectionVersion={}", collectionVersion); + } + versionedCollectionName = CollectionVersionUtils.buildVersionedNameString(unversionedCollectionName, collectionVersion, false); + versionedItemType = CollectionVersionUtils.buildVersionedNameString(itemType, collectionVersion, false); + if (logger.isTraceEnabled()) { + logger.trace("getVersionedCollectionName() - using versioned collection name: collectionName={} versionedCollectionName={} versionedItemType={}", + unversionedCollectionName, versionedCollectionName, versionedItemType); + } + } + } else { + // version was passed in + collectionVersion = CollectionVersionUtils.getCollectionVersion(versionedCollectionName); + } + collectionInfo.put(MAP_VERSIONED_COLLECTION_NAME_KEY, versionedCollectionName); + collectionInfo.put(MAP_UNVERSIONED_COLLECTION_NAME_KEY, unversionedCollectionName); + collectionInfo.put(MAP_VERSIONED_ITEM_TYPE_KEY, versionedItemType); + collectionInfo.put(MAP_UNVERSIONED_ITEM_TYPE_KEY, unversionedItemType); + collectionInfo.put(MAP_COLLECTION_VERSION_KEY, collectionVersion); + + return collectionInfo; + } + + public Service getService( String serviceType ) { return getService( serviceType, true ); } @@ -246,32 +298,15 @@ public class ServiceManager { return null; } - // use versionedCollectionName if appropriate - String versionedCollectionName = info.getCollectionName(); - String unversionedCollectionName = CollectionVersionUtil.getBaseCollectionName(versionedCollectionName); + Map<String,String> collectionInfo = getVersionedCollectionInfo(info.getCollectionName(), info.getItemType()); + String versionedCollectionName = collectionInfo.get(MAP_VERSIONED_COLLECTION_NAME_KEY); + String versionedItemType = collectionInfo.get(MAP_VERSIONED_ITEM_TYPE_KEY); if (logger.isTraceEnabled()) { logger.trace("getService: serviceType={} incoming collectionName={}", serviceType, versionedCollectionName); } - // if versioned collection name was passed in, use it, because it may be for an old version - if (versionedCollectionName.equals(unversionedCollectionName)) { - // no version passed in - CollectionVersionManager collectionVersionManager = cvmf.getInstance(new CollectionScopeImpl(applicationId, unversionedCollectionName)); - // always bypass collection version cache for now - String currentCollectionVersion = collectionVersionManager.getCollectionVersion(true); - - if (currentCollectionVersion != "") { - if (logger.isTraceEnabled()) { - logger.trace("getService: currentCollectionVersion={}", currentCollectionVersion); - } - versionedCollectionName = CollectionVersionUtil.buildVersionedNameString(unversionedCollectionName, currentCollectionVersion, false); - String versionedItemType = CollectionVersionUtil.buildVersionedNameString(info.getItemType(), currentCollectionVersion, false); - if (logger.isTraceEnabled()) { - logger.trace("getService() - using versioned collection name: collectionName={} versionedCollectionName={} versionedItemType={}", - unversionedCollectionName, versionedCollectionName, versionedItemType); - } - info = ServiceInfo.getVersionedServiceInfo(info, versionedCollectionName, versionedItemType); - } + if (!versionedCollectionName.equals(info.getCollectionName())) { + info = ServiceInfo.getVersionedServiceInfo(info, versionedCollectionName, versionedItemType); } Service service = getServiceInstance( info );
