usergrid git commit: Fix failing test to ensure we don't set the same timestamp for all entities inserted into the index (since we don't do actually do that). Also ensure it's a long as what's require
Repository: usergrid Updated Branches: refs/heads/USERGRID-1080 22beca2cb -> c652171f4 Fix failing test to ensure we don't set the same timestamp for all entities inserted into the index (since we don't do actually do that). Also ensure it's a long as what's required of the timestamps Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c652171f Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c652171f Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c652171f Branch: refs/heads/USERGRID-1080 Commit: c652171f476f6ad953c94328e0fb2f4ea0b3aa44 Parents: 22beca2 Author: Michael Russo Authored: Wed Oct 28 16:56:19 2015 -0700 Committer: Michael Russo Committed: Wed Oct 28 16:56:19 2015 -0700 -- .../persistence/index/impl/EntityIndexTest.java | 14 ++ 1 file changed, 6 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c652171f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java -- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java index 5243d5a..d6758dd 100644 --- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java @@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicLong; import com.google.common.base.Optional; import org.apache.usergrid.persistence.core.astyanax.CassandraFig; import org.apache.usergrid.persistence.index.*; +import org.apache.usergrid.persistence.model.field.*; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -48,10 +49,6 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils; import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.model.entity.Id; import org.apache.usergrid.persistence.model.entity.SimpleId; -import org.apache.usergrid.persistence.model.field.ArrayField; -import org.apache.usergrid.persistence.model.field.IntegerField; -import org.apache.usergrid.persistence.model.field.StringField; -import org.apache.usergrid.persistence.model.field.UUIDField; import org.apache.usergrid.persistence.model.util.EntityUtils; import org.apache.usergrid.persistence.model.util.UUIDGenerator; @@ -394,7 +391,6 @@ public class EntityIndexTest extends BaseIT { int numberOfEntities = 1000; int versionToSearchFor = numberOfEntities / 2; -IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", SearchEdge.NodeType.SOURCE, 1 ); UUID entityUUID = UUID.randomUUID(); Id entityId = new SimpleId( "mehCar" ); @@ -408,9 +404,11 @@ public class EntityIndexTest extends BaseIT { Entity[] entity = new Entity[numberOfEntities]; for(int i = 0; i < numberOfEntities; i++) { entity[i] = EntityIndexMapUtils.fromMap( entityMap ); -EntityUtils.setId( entity[i], entityId ); -EntityUtils.setVersion( entity[i], UUIDGenerator.newTimeUUID() ); -entity[i].setField( new UUIDField( IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID ) ); +EntityUtils.setId(entity[i], entityId); +EntityUtils.setVersion(entity[i], UUIDGenerator.newTimeUUID()); +entity[i].setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID)); + +IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", SearchEdge.NodeType.SOURCE, System.currentTimeMillis()*1000 ); //index the new entity. This is where the loop will be set to create like 100 entities. indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i] ).build()).subscribe();
usergrid git commit: Added rudimentary changes to get elastic search working in the query index. Still won't work for core.
Repository: usergrid Updated Branches: refs/heads/usergrid2.0es de8608d65 -> 96140d3b7 Added rudimentary changes to get elastic search working in the query index. Still won't work for core. Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/96140d3b Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/96140d3b Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/96140d3b Branch: refs/heads/usergrid2.0es Commit: 96140d3b7dd46e28d6dd70649f641f59c3e95772 Parents: de8608d Author: George Reyes Authored: Wed Oct 28 16:54:23 2015 -0700 Committer: George Reyes Committed: Wed Oct 28 16:54:23 2015 -0700 -- .../org/apache/usergrid/utils/IndexUtils.java | 3 +- stack/corepersistence/pom.xml | 2 +- .../usergrid/persistence/index/IndexFig.java| 2 +- .../index/impl/EsEntityIndexImpl.java | 184 ++- .../persistence/index/impl/EsProvider.java | 11 +- .../persistence/index/impl/EsQueryVistor.java | 98 -- .../index/impl/IndexRefreshCommandImpl.java | 6 +- .../index/impl/NoOpFilterBuilder.java | 8 +- .../index/impl/NoOpQueryBuilder.java| 6 +- .../impl/SearchRequestBuilderStrategy.java | 25 +-- .../persistence/index/impl/SortBuilder.java | 8 +- stack/log4j.properties | 32 12 files changed, 205 insertions(+), 180 deletions(-) -- http://git-wip-us.apache.org/repos/asf/usergrid/blob/96140d3b/stack/core/src/main/java/org/apache/usergrid/utils/IndexUtils.java -- diff --git a/stack/core/src/main/java/org/apache/usergrid/utils/IndexUtils.java b/stack/core/src/main/java/org/apache/usergrid/utils/IndexUtils.java index 5fc0f74..c22e5c3 100644 --- a/stack/core/src/main/java/org/apache/usergrid/utils/IndexUtils.java +++ b/stack/core/src/main/java/org/apache/usergrid/utils/IndexUtils.java @@ -53,7 +53,8 @@ public class IndexUtils { private static final Logger LOG = LoggerFactory.getLogger( IndexUtils.class ); -static Analyzer analyzer = new StandardAnalyzer( Version.LUCENE_30 ); +//TODO: GREY does this really need the VERSION.LATEST in the standar analyzer? doesn't seem to take it anymore +static Analyzer analyzer = new StandardAnalyzer( ); private static void buildKeyValueSet( Object node, Map> keyValues, String path, http://git-wip-us.apache.org/repos/asf/usergrid/blob/96140d3b/stack/corepersistence/pom.xml -- diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml index 776ad37..d9d60f3 100644 --- a/stack/corepersistence/pom.xml +++ b/stack/corepersistence/pom.xml @@ -59,7 +59,7 @@ limitations under the License. 3.2.1 2.4 3.1 -2.0.0-rc1 +2.0.0 3.1.3 18.0 4.0-beta5 http://git-wip-us.apache.org/repos/asf/usergrid/blob/96140d3b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java -- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java index e093d7d..5a34d7c 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java @@ -178,7 +178,7 @@ public interface IndexFig extends GuicyFig { * Return the type of Elasticsearch client. Valid values are NODE or TRANSPORT. */ @Key( ELASTICSEARCH_CLIENT_TYPE ) -@Default( "NODE") +@Default( "TRANSPORT") String getClientType(); /** http://git-wip-us.apache.org/repos/asf/usergrid/blob/96140d3b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java -- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 9be591e..069fd25 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -18,48 +18,30 @@ package org.apache.usergrid.persistence.index.impl; -import com.codahale.metrics.Meter; -import com
usergrid git commit: -Convert search in ES to use term queries instead of scroll with filters. -Fix delete so we remove connections from Elasticsearch as well as the entity on entity delete event.
Repository: usergrid Updated Branches: refs/heads/USERGRID-1080 [created] 22beca2cb -Convert search in ES to use term queries instead of scroll with filters. -Fix delete so we remove connections from Elasticsearch as well as the entity on entity delete event. Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/22beca2c Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/22beca2c Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/22beca2c Branch: refs/heads/USERGRID-1080 Commit: 22beca2cb3714833ddf6a748522c6bead17fe9f2 Parents: 2686054 Author: Michael Russo Authored: Wed Oct 28 15:58:41 2015 -0700 Committer: Michael Russo Committed: Wed Oct 28 15:58:41 2015 -0700 -- .../corepersistence/index/IndexServiceImpl.java | 2 +- .../persistence/index/CandidateResult.java | 11 +- .../persistence/index/EntityIndexBatch.java | 2 +- .../usergrid/persistence/index/IndexFig.java| 6 + .../index/impl/DeIndexOperation.java| 5 + .../index/impl/EsEntityIndexBatchImpl.java | 34 - .../index/impl/EsEntityIndexImpl.java | 149 +++ .../persistence/index/impl/IndexingUtils.java | 2 +- 8 files changed, 137 insertions(+), 74 deletions(-) -- http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java -- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java index ccb6221..b2a1a2a 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java @@ -219,7 +219,7 @@ public class IndexServiceImpl implements IndexService { //collect results into a single batch .collect( () -> ei.createBatch(), ( batch, candidateResult ) -> { logger.debug( "Deindexing on edge {} for entity {} added to batch",searchEdge , entityId ); -batch.deindex( searchEdge, candidateResult ); +batch.deindex( candidateResult ); } ) //return the future from the batch execution .flatMap( batch ->Observable.just(batch.build()) ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java -- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java index 2375509..9b1898c 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java @@ -32,10 +32,12 @@ import org.apache.usergrid.persistence.model.entity.Id; public class CandidateResult implements EntityVersion { private final Id entityId; private final UUID entityVersion; +private final String docId; -public CandidateResult( Id entityId, UUID entityVersion ) { +public CandidateResult( Id entityId, UUID entityVersion, String docId ) { this.entityId = entityId; this.entityVersion = entityVersion; +this.docId = docId; } @Override @@ -48,6 +50,10 @@ public class CandidateResult implements EntityVersion { return entityId; } +public String getDocId() { +return docId; +} + @Override public boolean equals( final Object o ) { @@ -66,6 +72,9 @@ public class CandidateResult implements EntityVersion { if ( !entityVersion.equals( that.entityVersion ) ) { return false; } +if ( !docId.equals( that.docId ) ) { +return false; +} return true; } http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java -- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java index 98652c1..17dd4d3 100644 --- a/stack/corepersistence/qu
usergrid git commit: Adds the ability to get credentials info as superuser for user migration
Repository: usergrid Updated Branches: refs/heads/USERGRID-1079-1x [created] 07e4cc211 Adds the ability to get credentials info as superuser for user migration Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/07e4cc21 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/07e4cc21 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/07e4cc21 Branch: refs/heads/USERGRID-1079-1x Commit: 07e4cc211ac715fca19b24355ca690bab2876218 Parents: afe0c51 Author: Todd Nine Authored: Wed Oct 28 15:27:36 2015 -0600 Committer: Todd Nine Committed: Wed Oct 28 15:27:36 2015 -0600 -- .../rest/applications/users/UserResource.java | 46 ++-- .../usergrid/management/ManagementService.java | 2 + .../cassandra/ManagementServiceImpl.java| 19 3 files changed, 63 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/usergrid/blob/07e4cc21/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java -- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java index ea17282..5fa67af 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java @@ -41,7 +41,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; + +import org.apache.amber.oauth2.common.exception.OAuthProblemException; +import org.apache.amber.oauth2.common.message.OAuthResponse; +import org.apache.commons.lang.StringUtils; + import org.apache.usergrid.management.ActivationState; +import org.apache.usergrid.persistence.CredentialsInfo; import org.apache.usergrid.persistence.EntityManager; import org.apache.usergrid.persistence.Identifier; import org.apache.usergrid.persistence.entities.User; @@ -50,13 +56,10 @@ import org.apache.usergrid.rest.ApiResponse; import org.apache.usergrid.rest.applications.ServiceResource; import org.apache.usergrid.rest.exceptions.RedirectionException; import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess; +import org.apache.usergrid.rest.security.annotations.RequireSystemAccess; import org.apache.usergrid.security.oauth.AccessInfo; import org.apache.usergrid.security.tokens.exceptions.TokenException; -import org.apache.amber.oauth2.common.exception.OAuthProblemException; -import org.apache.amber.oauth2.common.message.OAuthResponse; -import org.apache.commons.lang.StringUtils; - import com.sun.jersey.api.json.JSONWithPadding; import com.sun.jersey.api.view.Viewable; @@ -159,6 +162,41 @@ public class UserResource extends ServiceResource { return new JSONWithPadding( response, callback ); } +@GET +@RequireSystemAccess +@Path("password") +public JSONWithPadding getUserPassword(@QueryParam("callback") @DefaultValue("callback") String callback ) +throws Exception { + +logger.info( "UserResource.setUserPassword" ); + + +final ApiResponse response = createApiResponse(); +response.setAction( "get user password" ); + +final UUID applicationId = getApplicationId(); +final UUID targetUserId = getUserUuid(); + +if ( applicationId == null ) { +response.setError( "Application not found" ); +return new JSONWithPadding( response, callback ); +} + +if ( targetUserId == null ) { +response.setError( "User not found" ); +return new JSONWithPadding( response, callback ); +} + +final CredentialsInfo credentialsInfo = management.getAppUserPasswordRaw( applicationId, targetUserId ); + + +response.setProperty( "credentials", credentialsInfo ); + + +return new JSONWithPadding( response, callback ); +} + + @POST @Path("password") http://git-wip-us.apache.org/repos/asf/usergrid/blob/07e4cc21/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java -- diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java index 708e823..b5537ec 100644 --- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java +++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java @@ -283,6 +283,8 @@ public interface ManagementServ
usergrid git commit: Started the migration to use elastic search 2.0
Repository: usergrid Updated Branches: refs/heads/usergrid2.0es [created] de8608d65 Started the migration to use elastic search 2.0 Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/de8608d6 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/de8608d6 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/de8608d6 Branch: refs/heads/usergrid2.0es Commit: de8608d65cec53b399bd65cd7d121fe43fc6b09d Parents: 2686054 Author: George Reyes Authored: Wed Oct 28 13:54:26 2015 -0700 Committer: George Reyes Committed: Wed Oct 28 13:54:26 2015 -0700 -- stack/corepersistence/pom.xml | 2 +- stack/corepersistence/queryindex/pom.xml | 9 - .../usergrid/persistence/index/impl/EsProvider.java | 13 +++-- .../persistence/index/query/tree/QueryVisitor.java| 14 -- 4 files changed, 12 insertions(+), 26 deletions(-) -- http://git-wip-us.apache.org/repos/asf/usergrid/blob/de8608d6/stack/corepersistence/pom.xml -- diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml index 4e4648e..776ad37 100644 --- a/stack/corepersistence/pom.xml +++ b/stack/corepersistence/pom.xml @@ -59,7 +59,7 @@ limitations under the License. 3.2.1 2.4 3.1 -1.4.4 +2.0.0-rc1 3.1.3 18.0 4.0-beta5 http://git-wip-us.apache.org/repos/asf/usergrid/blob/de8608d6/stack/corepersistence/queryindex/pom.xml -- diff --git a/stack/corepersistence/queryindex/pom.xml b/stack/corepersistence/queryindex/pom.xml index 2484cfd..e8a2336 100644 --- a/stack/corepersistence/queryindex/pom.xml +++ b/stack/corepersistence/queryindex/pom.xml @@ -92,15 +92,6 @@ org.elasticsearch elasticsearch ${elasticsearch.version} - - - - -antlr-runtime -org.antlr - - - http://git-wip-us.apache.org/repos/asf/usergrid/blob/de8608d6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java -- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java index 096bd17..d94dbae 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java @@ -20,11 +20,11 @@ package org.apache.usergrid.persistence.index.impl; import java.net.InetAddress; +import java.net.InetSocketAddress; import java.net.UnknownHostException; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.node.Node; @@ -123,7 +123,7 @@ public class EsProvider { final String clusterName = indexFig.getClusterName(); final int port = indexFig.getPort(); -ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put( "cluster.name", clusterName ) +Settings.Builder settings = Settings.settingsBuilder().put( "cluster.name", clusterName ) .put( "client.transport.sniff", true ); String nodeName = indexFig.getNodeName(); @@ -141,15 +141,16 @@ public class EsProvider { settings.put( "node.name", nodeName ); - -TransportClient transportClient = new TransportClient( settings.build() ); +TransportClient.Builder transportClient = new TransportClient.Builder().settings( settings.build() ); //new TransportClient( settings.build() ); // we will connect to ES on all configured hosts for ( String host : indexFig.getHosts().split( "," ) ) { +InetSocketAddress inetSocketAddress = new InetSocketAddress(host ,port ); +settings.put( new InetSocketTransportAddress( inetSocketAddress ) ); transportClient.addTransportAddress( new InetSocketTransportAddress( host, port ) ); } -return transportClient; +return transportClient.build(); } @@ -181,7 +182,7 @@ public class EsProvider { final String hostString = hos
usergrid git commit: Added the collection tool that needs to be expanded
Repository: usergrid Updated Branches: refs/heads/userCollectionFix [created] a72da10f8 Added the collection tool that needs to be expanded Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a72da10f Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a72da10f Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a72da10f Branch: refs/heads/userCollectionFix Commit: a72da10f85bd01b47950e18b4ab2f47dfa4ecf23 Parents: afe0c51 Author: George Reyes Authored: Wed Oct 28 11:23:37 2015 -0700 Committer: George Reyes Committed: Wed Oct 28 11:23:37 2015 -0700 -- .../rest/applications/users/UsersResource.java | 3 + ...cateUniquePropertyExistsExceptionMapper.java | 6 + .../usergrid/tools/CollectionUserFix.java | 255 +++ 3 files changed, 264 insertions(+) -- http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java -- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java index 6325d5a..408299a 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java @@ -100,6 +100,9 @@ public class UsersResource extends ServiceResource { @Path("{itemName}") public AbstractContextResource addNameParameter( @Context UriInfo ui, @PathParam("itemName") PathSegment itemName ) throws Exception { +//TODO: read repair if entity ref doesn't match returned entity. Detect index corruption and delete reference if +//reference isn't correct. + logger.info( "ServiceResource.addNameParameter" ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java -- diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java index b910b58..0ea1567 100644 --- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java +++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java @@ -29,6 +29,12 @@ import static javax.ws.rs.core.Response.Status.BAD_REQUEST; public class DuplicateUniquePropertyExistsExceptionMapper extends AbstractExceptionMapper { +//when you get this exception then fire a repair task that will grab a thread from a thread pool and async run the repair task +// That task will then go through and double checkk all the entities and verify that their ref's and entities match up. + + +//npe, delete entity ref +//mismatched pointers, update the entity ref with whatever is in the collection. ( retrieve via query ) . @Override public Response toResponse( DuplicateUniquePropertyExistsException e ) { return toResponse( BAD_REQUEST, e ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java -- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java new file mode 100644 index 000..829487a --- /dev/null +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.tools;
usergrid git commit: USERGRID-1044: don't log UUID mismatch if UUID was never retrieved during load
Repository: usergrid Updated Branches: refs/heads/USERGRID-1044 2a09bf15d -> 0fc18041e USERGRID-1044: don't log UUID mismatch if UUID was never retrieved during load Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0fc18041 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0fc18041 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0fc18041 Branch: refs/heads/USERGRID-1044 Commit: 0fc18041e32df049040f26a0faaef1f46dbe6d9f Parents: 2a09bf1 Author: Mike Dunker Authored: Wed Oct 28 08:09:19 2015 -0700 Committer: Mike Dunker Committed: Wed Oct 28 08:09:19 2015 -0700 -- .../scala/org/apache/usergrid/scenarios/AuditScenarios.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/usergrid/blob/0fc18041/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala -- diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala index d1e2c4b..cc59e0a 100644 --- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala +++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala @@ -183,7 +183,7 @@ object AuditScenarios { errorString += "|NoUuidReturned" Settings.incAuditPayloadUuidError() println(s"PAYLOAD UUID MISSING (DIRECT): requestedUuid=$uuid") -} else if (!uuid.equalsIgnoreCase(entityUuid)) { +} else if (!uuid.isEmpty && !uuid.equalsIgnoreCase(entityUuid)) { errorString += "|ReturnedUuidMismatch" Settings.incAuditPayloadUuidError() println(s"PAYLOAD UUID MISMATCH (DIRECT): requestedUuid=$uuid returnedUuid=$entityUuid") @@ -247,7 +247,7 @@ object AuditScenarios { errorString += "|NoUuidReturned" Settings.incAuditPayloadUuidError() println(s"PAYLOAD UUID MISSING (QUERY): requestedUuid=$uuid") - } else if (!uuid.equalsIgnoreCase(entityUuid)) { + } else if (!uuid.isEmpty && !uuid.equalsIgnoreCase(entityUuid)) { errorString += "|ReturnedUuidMismatch" Settings.incAuditPayloadUuidError() println(s"PAYLOAD UUID MISMATCH (QUERY): requestedUuid=$uuid returnedUuid=$entityUuid")
usergrid git commit: USERGRID-1044: print progress during audit and log failures as such
Repository: usergrid Updated Branches: refs/heads/USERGRID-1044 2e70a72b9 -> 2a09bf15d USERGRID-1044: print progress during audit and log failures as such Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2a09bf15 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2a09bf15 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2a09bf15 Branch: refs/heads/USERGRID-1044 Commit: 2a09bf15d8063199bf25b7e7f437929601f7193a Parents: 2e70a72 Author: Mike Dunker Authored: Wed Oct 28 07:28:51 2015 -0700 Committer: Mike Dunker Committed: Wed Oct 28 07:28:51 2015 -0700 -- .../org/apache/usergrid/datagenerators/FeederGenerator.scala | 8 .../scala/org/apache/usergrid/scenarios/AuditScenarios.scala | 6 +++--- .../usergrid/scenarios/EntityCollectionScenarios.scala | 8 +--- 3 files changed, 16 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/usergrid/blob/2a09bf15/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala -- diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala index 38db19a..a9225b2 100755 --- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala +++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala @@ -255,6 +255,9 @@ object FeederGenerator { val csvLines = if (Settings.feedAuditUuids) Source.fromFile(Settings.feedAuditUuidFilename).getLines().toArray else Array[String]() val csvLinesLen = csvLines.length var counter = new AtomicInteger(0) +var entityCounter: Long = 0L +var lastEntityCountPrinted: Long = 0L +var entityProgressCount: Long = Settings.entityProgressCount override def hasNext: Boolean = true @@ -284,6 +287,11 @@ object FeederGenerator { val accessField = if (uuid != "") uuid else name val queryField = if (uuid != "") s"uuid='$uuid'" else s"name='$name'" + if (entityProgressCount > 0L && entityCounter >= lastEntityCountPrinted + entityProgressCount) { +println(s"CSV Entity: $entityCounter") +lastEntityCountPrinted = entityCounter + } + Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified, "lastStatus" -> lastStatus, "validEntity" -> validEntity, "accessField" -> accessField, "queryField" -> queryField) } http://git-wip-us.apache.org/repos/asf/usergrid/blob/2a09bf15/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala -- diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala index 1ce2bc2..d1e2c4b 100644 --- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala +++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala @@ -202,15 +202,15 @@ object AuditScenarios { Settings.incAuditNotFoundViaQuery() println(s"NOT FOUND VIA QUERY: $collectionName.$reqName ($uuid)") } - session + session.markAsFailed } else if (saveFailures) { Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure", lastStatus) - session + session.markAsFailed } else { session.markAsFailed } - }).exitHereIfFailed + }) val getCollectionEntity = exec( http("GET collection entity") http://git-wip-us.apache.org/repos/asf/usergrid/blob/2a09bf15/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala -- diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala index d5b3101..22b9f08 100644 --- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala +++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala @@ -241,16 +241,18 @@ object EntityCollectionScenarios { val entityName = session("entityName").as[String] val modified = if (status == 200) session(SessionVarModified).as[Long] else 0 val collectionName = session("collectionName").as[String] +Settings.addUuid(uuid, collectionName