usergrid git commit: First and untested stab at multi-tenant migration.

2015-10-29 Thread snoopdave 
Repository: usergrid
Updated Branches:
  refs/heads/multitenant-migration [created] d79bf4c36


First and untested stab at multi-tenant migration.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d79bf4c3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d79bf4c3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d79bf4c3

Branch: refs/heads/multitenant-migration
Commit: d79bf4c36810fdab578a86e4de0ae725f1aa6e75
Parents: 2686054
Author: Dave Johnson 
Authored: Thu Oct 29 08:56:14 2015 -0400
Committer: Dave Johnson 
Committed: Thu Oct 29 08:56:14 2015 -0400

--
 stack/scripts/create_test_data.py| 213 +
 stack/scripts/migrate_entity_data.py | 301 +-
 2 files changed, 429 insertions(+), 85 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/d79bf4c3/stack/scripts/create_test_data.py
--
diff --git a/stack/scripts/create_test_data.py 
b/stack/scripts/create_test_data.py
new file mode 100644
index 000..de85da0
--- /dev/null
+++ b/stack/scripts/create_test_data.py
@@ -0,0 +1,213 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+import logging
+from logging.handlers import RotatingFileHandler
+import argparse
+import time
+import requests
+import json
+
+# Creates two organizations each with two apps each with three collections 
each with 100 entities
+# Caller must provide a "slug" string which will be used as a prefix for all 
names
+#
+# For example, if the slug is mytest then:
+#   Orgs will be named mytest_org0 and mytest_org1
+#   Apps will be named mytest_org0_app0 and so on
+#   Collections will be named mytest_org0_app0_col0 and so on
+#   Entities will be named mytest_org0_app0_col0_entity and so on
+#   Org admins will be named mytest_org0_admin and mytest_org1_admin (both 
with password test) 
+
+def parse_args():
+parser = argparse.ArgumentParser(description="Usergrid Test Data Creation 
Tool")
+
+parser.add_argument("--endpoint",
+help="The endpoint to use for making API requests.",
+type=str,
+default="http://localhost:8080;)
+
+parser.add_argument("--user",
+help="System Admin Credentials used to authenticate 
with Usergrid  ",
+type=str,
+required=True)
+
+parser.add_argument("--slug",
+help="Unique string to be used to name organization, 
applications and other things create",
+type=str,
+required=True)
+
+my_args = parser.parse_args(sys.argv[1:])
+
+arg_vars = vars(my_args)
+creds = arg_vars["user"].split(":")
+if len(creds) != 2:
+print("Credentials not properly specified.  Must be '-u '. 
Exiting...")
+exit_on_error()
+else:
+arg_vars["user"] = creds[0]
+arg_vars["pass"] = creds[1]
+
+return arg_vars
+
+
+class Creator:
+def __init__(self):
+self.args = parse_args()
+self.endpoint = self.args["endpoint"]
+self.logger = init_logging(self.__class__.__name__)
+self.admin_user = self.args["user"]
+self.admin_pass = self.args["pass"]
+self.slug = self.args["slug"]
+
+def run(self):
+self.logger.info("Initializing...")
+
+if not self.is_endpoint_available():
+exit_on_error("Endpoint is not available, aborting")
+
+for orgIndex in range(2):
+orgName = self.slug + "_org" + str(orgIndex)
+orgUser = orgName + "_admin"
+orgEmail = orgUser + "@example.com"
+
+url = self.endpoint + "/management/orgs"
+body = json.dumps({"username":orgUser, "email":orgEmail, 
"password":"test", "organization":orgName })
+r = requests.post(url=url, data=body, auth=(self.admin_user, 
self.admin_pass))
+if

usergrid git commit: Added --super and --admin so we can specify both superuser and org admin credentials. Did some testing and migration seems to work for one tenant, have not tried two yet. Commente

2015-10-29 Thread snoopdave 
Repository: usergrid
Updated Branches:
  refs/heads/multitenant-migration d79bf4c36 -> 471dc359d


Added --super and --admin so we can specify both superuser and org admin 
credentials.
Did some testing and migration seems to work for one tenant, have not tried two 
yet.
Commented out the de-dup calls because de-dup never finishes.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/471dc359
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/471dc359
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/471dc359

Branch: refs/heads/multitenant-migration
Commit: 471dc359d5920ea521c4058e7829a37494a17041
Parents: d79bf4c
Author: Dave Johnson 
Authored: Thu Oct 29 10:07:33 2015 -0400
Committer: Dave Johnson 
Committed: Thu Oct 29 10:07:33 2015 -0400

--
 stack/scripts/migrate_entity_data.py | 130 --
 1 file changed, 87 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/471dc359/stack/scripts/migrate_entity_data.py
--
diff --git a/stack/scripts/migrate_entity_data.py 
b/stack/scripts/migrate_entity_data.py
index f576108..0edd319 100644
--- a/stack/scripts/migrate_entity_data.py
+++ b/stack/scripts/migrate_entity_data.py
@@ -20,7 +20,9 @@
 #
 # STEP 1 - SETUP TENANT ONE TOMCAT RUNNING 2.1 NOT IN SERVICE AND INIT 
MIGRATION
 #
-#   python migrate_entity_data.py --org  --user 
: --init
+#   Login to the Tomcat instance and run this command, specifying both 
superuser and tenant organization admin creds:
+#
+#   python migrate_entity_data.py --org  --super : 
--admin : --init
 #
 #   This command will setup and bootstrap the database, setup the migration 
system and update index mappings:
 #   - /system/database/setup
@@ -31,24 +33,34 @@
 #   Then it will migrate appinfos, re-index the management app and then for 
each of the specified org's apps
 #   it will de-dup connections and re-index the app.
 #
+#   Write down the 'Re-index start' timestamp when this is finished.
+#
 # STEP 2 - PUT TENANT ONE TOMCATS IN SERVICE AND DO DELTA MIGRATION
 #
-#   python migrate_entity_data.py --org  --user 
: --date
+#   On the same Tomcat instance and run this command with the --date timestamp 
you noted in the previous step:
+#
+#   python migrate_entity_data.py --org  --super : 
--admin : --date 
 #
 #   Then it will migrate appinfos, re-index the management app and then for 
each of the specified org's apps
 #   it will de-dup connections and re-index the app with a start-date 
specified so only data modified since
 #   STEP 1 will be re-indexed.
 #
-# STEP 3 - SETUP TENENT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
+# STEP 3 - SETUP TENANT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
+#
+#   Login to the Tomcat instance and run this command, specifying both 
superuser and tenant organization admin creds:
 #
-#   python migrate_entity_data.py --org  --user 
: --date
+#   python migrate_entity_data.py --org  --super : 
--admin :
 #
 #   This command will migrate appinfos, re-index the management app and then 
for each of the specified org's apps
 #   it will de-dup connections and re-index the app.
 #
+#   Write down the 'Re-index start' timestamp when this is finished.
+
 # STEP 4 - PUT TENANT TWO TOMCATS IN SERVICE AND DO DELTA MIGRATION
 #
-#   python migrate_entity_data.py --org  --user 
: --date
+#   On the same Tomcat instance and run this command with the --date timestamp 
you noted in the previous step:
+#
+#   python migrate_entity_data.py --org  --super : 
--admin : --date 
 #
 #   Then it will migrate appinfos, re-index the management app and then for 
each of the specified org's apps
 #   it will de-dup connections and re-index the app with a start-date 
specified so only data modified since
@@ -56,7 +68,9 @@
 #
 # STEP 5 - FULL DATA MIGRATION
 #
-#   python migrate_entity_data.py --user : --full
+#   Login to any Tomcat instance in the cluster and run this command (admin 
user creds must be specificed but will be ignored):
+#
+#   python migrate_entity_data.py --super : --admin 
: --full
 #
 #   This command will run the full data migration.
 #
@@ -87,6 +101,8 @@ PLUGIN_ENTITYDATA = 'collections-entity-data'
 PLUGIN_INDEX_MAPPING = 'index_mapping_migration'
 PLUGIN_CORE_DATA = 'core-data'
 
+MANAGEMENT_APP_ID = 'b6768a08-b5d5-11e3-a495-11ddb1de66c8'
+
 
 
 def parse_args():
@@ -97,8 +113,13 @@ def parse_args():
 type=str,
 default='http://localhost:8080')
 
-parser.add_argument('--user',
-help='System Admin Credentials used to authenticate 
with Usergrid  ',
+parser.add_argument('--super',
+help='Superuser

usergrid git commit: There seems to be an error with the content that is being pushed to elastic search. Something about compressor

2015-10-29 Thread grey 
Repository: usergrid
Updated Branches:
  refs/heads/usergrid2.0es 96140d3b7 -> bb23c3a26


There seems to be an error with the content that is being pushed to elastic 
search. Something about compressor


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bb23c3a2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bb23c3a2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bb23c3a2

Branch: refs/heads/usergrid2.0es
Commit: bb23c3a26ef0bfe4adf334a3c4b3fcb864c426ab
Parents: 96140d3
Author: George Reyes 
Authored: Thu Oct 29 09:53:12 2015 -0700
Committer: George Reyes 
Committed: Thu Oct 29 09:53:12 2015 -0700

--
 .../usergrid/persistence/index/impl/EsEntityIndexImpl.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/bb23c3a2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
--
diff --git 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 069fd25..2018ea0 100644
--- 
a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ 
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -39,6 +39,7 @@ import 
org.elasticsearch.action.search.SearchScrollRequestBuilder;
 import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilderString;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
@@ -306,9 +307,9 @@ public class EsEntityIndexImpl implements 
EntityIndex,VersionedData {
 
 final RetryOperation retryOperation = () -> {
 final String tempId = UUIDGenerator.newTimeUUID().toString();
-
+XContentBuilderString xContentBuilderString = new 
XContentBuilderString( "hi" );
 esProvider.getClient().prepareIndex( alias.getWriteAlias(), 
VERIFY_TYPE, tempId )
- .setSource(DEFAULT_PAYLOAD).get();
+ .setSource(xContentBuilderString).get();
 
 logger.info( "Successfully created new document with docId {} "
  + "in index read {} write {} and type {}",

[2/2] usergrid git commit: Fix failing test to ensure we don't set the same timestamp for all entities inserted into the index (since we don't do actually do that). Also ensure it's a long as what's r

2015-10-29 Thread mdunker 
Fix failing test to ensure we don't set the same timestamp for all entities 
inserted into the index (since we don't do actually do that).  Also ensure it's 
a long as what's required of the timestamps


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c652171f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c652171f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c652171f

Branch: refs/heads/2.1-release
Commit: c652171f476f6ad953c94328e0fb2f4ea0b3aa44
Parents: 22beca2
Author: Michael Russo 
Authored: Wed Oct 28 16:56:19 2015 -0700
Committer: Michael Russo 
Committed: Wed Oct 28 16:56:19 2015 -0700

--
 .../persistence/index/impl/EntityIndexTest.java   | 14 ++
 1 file changed, 6 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c652171f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
--
diff --git 
a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
 
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 5243d5a..d6758dd 100644
--- 
a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ 
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import com.google.common.base.Optional;
 import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
 import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.model.field.*;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -48,10 +49,6 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.field.ArrayField;
-import org.apache.usergrid.persistence.model.field.IntegerField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
 import org.apache.usergrid.persistence.model.util.EntityUtils;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 
@@ -394,7 +391,6 @@ public class EntityIndexTest extends BaseIT {
 int numberOfEntities = 1000;
 int versionToSearchFor = numberOfEntities / 2;
 
-IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", 
SearchEdge.NodeType.SOURCE, 1 );
 
 UUID entityUUID = UUID.randomUUID();
 Id entityId = new SimpleId( "mehCar" );
@@ -408,9 +404,11 @@ public class EntityIndexTest extends BaseIT {
 Entity[] entity = new Entity[numberOfEntities];
 for(int i = 0; i < numberOfEntities; i++) {
 entity[i] = EntityIndexMapUtils.fromMap( entityMap );
-EntityUtils.setId( entity[i], entityId );
-EntityUtils.setVersion( entity[i], UUIDGenerator.newTimeUUID() );
-entity[i].setField( new UUIDField( 
IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID ) );
+EntityUtils.setId(entity[i], entityId);
+EntityUtils.setVersion(entity[i], UUIDGenerator.newTimeUUID());
+entity[i].setField(new 
UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID));
+
+IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", 
SearchEdge.NodeType.SOURCE, System.currentTimeMillis()*1000 );
 
 //index the new entity. This is where the loop will be set to 
create like 100 entities.
 indexProducer.put(entityIndex.createBatch().index( searchEdge, 
entity[i]  ).build()).subscribe();

[2/3] usergrid git commit: Updated call for new read api in 2.0

2015-10-29 Thread toddnine 
Updated call for new read api in 2.0


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7fc7c55b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7fc7c55b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7fc7c55b

Branch: refs/heads/USERGRID-1079-2x
Commit: 7fc7c55b7839ff9ba5a71b1c3d74a095c27f49ff
Parents: 08e9781
Author: Todd Nine 
Authored: Thu Oct 29 13:56:14 2015 -0600
Committer: Todd Nine 
Committed: Thu Oct 29 13:56:14 2015 -0600

--
 .../apache/usergrid/rest/applications/users/UserResource.java| 4 ++--
 .../java/org/apache/usergrid/management/ManagementService.java   | 2 +-
 .../usergrid/management/cassandra/ManagementServiceImpl.java | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/7fc7c55b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index fb10245..716f367 100644
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -169,7 +169,7 @@ public class UserResource extends ServiceResource {
 
 @GET
 @RequireSystemAccess
-@Path("password")
+@Path("credentials")
 public JSONWithPadding getUserPassword(@QueryParam("callback") 
@DefaultValue("callback") String callback )
 throws Exception {
 
@@ -192,7 +192,7 @@ public class UserResource extends ServiceResource {
 return new JSONWithPadding( response, callback );
 }
 
-final CredentialsInfo credentialsInfo = 
management.getAppUserPasswordRaw( applicationId, targetUserId );
+final CredentialsInfo credentialsInfo = 
management.getAppUserCredentialsInfo( applicationId, targetUserId );
 
 
 response.setProperty( "credentials", credentialsInfo );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/7fc7c55b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
 
b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index 3f02e5a..cf2924b 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -288,7 +288,7 @@ public interface ManagementService {
 public void setAppUserPassword( UUID applicationId, UUID userId, String 
oldPassword, String newPassword )
 throws Exception;
 
-CredentialsInfo getAppUserPasswordRaw( final UUID applicationId, final 
UUID userId ) throws Exception;
+CredentialsInfo getAppUserCredentialsInfo( final UUID applicationId, final 
UUID userId ) throws Exception;
 
 /**
  * Set the credentials info into the

http://git-wip-us.apache.org/repos/asf/usergrid/blob/7fc7c55b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
 
b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index b633727..dfd0cb1 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -2825,7 +2825,7 @@ public class ManagementServiceImpl implements 
ManagementService {
 
 
 @Override
-public CredentialsInfo getAppUserPasswordRaw( final UUID applicationId, 
final UUID userId ) throws Exception {
+public CredentialsInfo getAppUserCredentialsInfo( final UUID 
applicationId, final UUID userId ) throws Exception {
 
 final User user = emf.getEntityManager( applicationId ).get( userId, 
User.class );
 
@@ -2833,7 +2833,7 @@ public class ManagementServiceImpl implements 
ManagementService {
 throw new EntityNotFoundException("Could not find user with id " + 
userId + " in application" + applicationId  );
 }
 
-final CredentialsInfo ci = readUserPasswordCredentials( applicationId, 
userId );
+final CredentialsInfo ci = readUserPasswordCredentials( applicationId, 
userId,

[1/3] usergrid git commit: Adds the ability to get credentials info as superuser for user migration

2015-10-29 Thread toddnine 
Repository: usergrid
Updated Branches:
  refs/heads/USERGRID-1079-2x 5ed8c7ce1 -> 4784b34f3


Adds the ability to get credentials info as superuser for user migration


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/08e97813
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/08e97813
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/08e97813

Branch: refs/heads/USERGRID-1079-2x
Commit: 08e978132375751caf90ebcffde7a20d5e84b206
Parents: 5ed8c7c
Author: Todd Nine 
Authored: Wed Oct 28 15:27:36 2015 -0600
Committer: Todd Nine 
Committed: Thu Oct 29 13:55:38 2015 -0600

--
 .../rest/applications/users/UserResource.java   | 45 ++--
 .../usergrid/management/ManagementService.java  |  2 +
 .../cassandra/ManagementServiceImpl.java| 19 +
 3 files changed, 62 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/08e97813/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index df88cf0..fb10245 100644
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -42,6 +42,11 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.annotation.Scope;
 import org.springframework.stereotype.Component;
+
+import org.apache.amber.oauth2.common.exception.OAuthProblemException;
+import org.apache.amber.oauth2.common.message.OAuthResponse;
+import org.apache.commons.lang.StringUtils;
+
 import org.apache.usergrid.management.ActivationState;
 import org.apache.usergrid.persistence.CredentialsInfo;
 import org.apache.usergrid.persistence.EntityManager;
@@ -52,13 +57,10 @@ import org.apache.usergrid.rest.ApiResponse;
 import org.apache.usergrid.rest.applications.ServiceResource;
 import org.apache.usergrid.rest.exceptions.RedirectionException;
 import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
 import org.apache.usergrid.security.oauth.AccessInfo;
 import org.apache.usergrid.security.tokens.exceptions.TokenException;
 
-import org.apache.amber.oauth2.common.exception.OAuthProblemException;
-import org.apache.amber.oauth2.common.message.OAuthResponse;
-import org.apache.commons.lang.StringUtils;
-
 import com.sun.jersey.api.json.JSONWithPadding;
 import com.sun.jersey.api.view.Viewable;
 
@@ -165,6 +167,41 @@ public class UserResource extends ServiceResource {
 return new JSONWithPadding( response, callback );
 }
 
+@GET
+@RequireSystemAccess
+@Path("password")
+public JSONWithPadding getUserPassword(@QueryParam("callback") 
@DefaultValue("callback") String callback )
+throws Exception {
+
+logger.info( "UserResource.setUserPassword" );
+
+
+final ApiResponse response = createApiResponse();
+response.setAction( "get user password" );
+
+final UUID applicationId = getApplicationId();
+final UUID targetUserId = getUserUuid();
+
+if ( applicationId == null ) {
+response.setError( "Application not found" );
+return new JSONWithPadding( response, callback );
+}
+
+if ( targetUserId == null ) {
+response.setError( "User not found" );
+return new JSONWithPadding( response, callback );
+}
+
+final CredentialsInfo credentialsInfo = 
management.getAppUserPasswordRaw( applicationId, targetUserId );
+
+
+response.setProperty( "credentials", credentialsInfo );
+
+
+return new JSONWithPadding( response, callback );
+}
+
+
 
 @PUT
 @Path("credentials")

http://git-wip-us.apache.org/repos/asf/usergrid/blob/08e97813/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
--
diff --git 
a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
 
b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index d69de2e..3f02e5a 100644
--- 
a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ 
b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -288,6 +288,8 @@ public interface ManagementService {
 public void setAppUserPassword( UUID applicationId,

[2/3] usergrid git commit: Corrections to some comments.

2015-10-29 Thread snoopdave 
Corrections to some comments.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/57a613b5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/57a613b5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/57a613b5

Branch: refs/heads/multitenant-migration
Commit: 57a613b58c4cd886c518eaa63e4e8179b3a19cb0
Parents: f8c703c
Author: Dave Johnson 
Authored: Thu Oct 29 17:58:10 2015 -0400
Committer: Dave Johnson 
Committed: Thu Oct 29 17:58:10 2015 -0400

--
 stack/scripts/create_test_data.py | 23 ++-
 1 file changed, 14 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/57a613b5/stack/scripts/create_test_data.py
--
diff --git a/stack/scripts/create_test_data.py 
b/stack/scripts/create_test_data.py
index de85da0..ef07254 100644
--- a/stack/scripts/create_test_data.py
+++ b/stack/scripts/create_test_data.py
@@ -27,11 +27,16 @@ import json
 # Caller must provide a "slug" string which will be used as a prefix for all 
names
 #
 # For example, if the slug is mytest then:
-#   Orgs will be named mytest_org0 and mytest_org1
-#   Apps will be named mytest_org0_app0 and so on
+#
+#   Orgs will be namedmytest_org0 and mytest_org1
+#   Org admins will be named  mytest_org0_admin and mytest_org1_admin (both 
with password test)
+#
+#   Apps will be namedmytest_org0_app0, mytest_org0_app1 and so on
 #   Collections will be named mytest_org0_app0_col0 and so on
-#   Entities will be named mytest_org0_app0_col0_entity and so on
-#   Org admins will be named mytest_org0_admin and mytest_org1_admin (both 
with password test) 
+#   Entities will be namedmytest_org0_app0_col0_entity and so on
+#
+# All entities in collection 0 will be connected to entities in collection 1.
+# All entities in collection 1 will be connected to entities in collection 2.
 
 def parse_args():
 parser = argparse.ArgumentParser(description="Usergrid Test Data Creation 
Tool")
@@ -42,7 +47,7 @@ def parse_args():
 default="http://localhost:8080;)
 
 parser.add_argument("--user",
-help="System Admin Credentials used to authenticate 
with Usergrid  ",
+help="Superuser credentials used to authenticate with 
Usergrid  ",
 type=str,
 required=True)
 
@@ -56,7 +61,7 @@ def parse_args():
 arg_vars = vars(my_args)
 creds = arg_vars["user"].split(":")
 if len(creds) != 2:
-print("Credentials not properly specified.  Must be '-u '. 
Exiting...")
+print("Credentials not properly specified.  Must be '--user 
'. Exiting...")
 exit_on_error()
 else:
 arg_vars["user"] = creds[0]
@@ -100,7 +105,7 @@ class Creator:
 if ( r.status_code != 200 ):
 print "Error logging into organization " + orgName + ": " + 
r.text
 return
-
+
 accessToken = r.json()["access_token"]
 
 for appIndex in range(2):
@@ -115,11 +120,11 @@ class Creator:
 
 print "   Created app: " + orgName + "/" + appName
 appUrl = self.endpoint + "/" + orgName + "/" + appName
-time.sleep(2) 
+time.sleep(2)
 
 for userIndex in range(2):
 userName = appName + "_user" + str(userIndex)
-email = userName + "@example.com" 
+email = userName + "@example.com"
 
 url = appUrl + "/users?access_token=" + accessToken
 body = json.dumps({"name":userName, "username":userName, 
"email":email, "password":"test"})

[1/3] usergrid git commit: Ensure that status is updated properly.

2015-10-29 Thread snoopdave 
Repository: usergrid
Updated Branches:
  refs/heads/multitenant-migration 471dc359d -> 8af152e6b


Ensure that status is updated properly.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f8c703c0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f8c703c0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f8c703c0

Branch: refs/heads/multitenant-migration
Commit: f8c703c02c1182ad63ad86587749eb1ae09c202a
Parents: 471dc35
Author: Dave Johnson 
Authored: Thu Oct 29 17:57:32 2015 -0400
Committer: Dave Johnson 
Committed: Thu Oct 29 17:57:32 2015 -0400

--
 .../rest/system/ConnectionResource.java | 48 +---
 1 file changed, 32 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/f8c703c0/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
--
diff --git 
a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
 
b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
index 6e683ed..14b79f3 100644
--- 
a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
+++ 
b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
@@ -55,6 +55,7 @@ import com.google.common.base.Preconditions;
 import com.sun.jersey.api.json.JSONWithPadding;
 
 import rx.Observable;
+import rx.functions.Action1;
 import rx.schedulers.Schedulers;
 
 
@@ -146,36 +147,51 @@ public class ConnectionResource extends 
AbstractContextResource {
 
 //start de duping and run in the background
 connectionService.deDupeConnections( applicationScopeObservable 
).buffer( 10, TimeUnit.SECONDS, 1000 )
- .doOnNext( buffer -> {
+ .doOnNext(buffer -> {
 
 
- final long runningTotal = count.addAndGet( 
buffer.size() );
+ final long runningTotal = 
count.addAndGet(buffer.size());
 
  final Map status = new 
HashMap() {{
- put( "countProcessed", runningTotal );
- put( "updatedTimestamp", 
System.currentTimeMillis() );
+ put("countProcessed", runningTotal);
+ put("updatedTimestamp", 
System.currentTimeMillis());
  }};
 
- statusService.setStatus( 
CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
- StatusService.Status.INPROGRESS, status 
).toBlocking().lastOrDefault( null );
- } ).doOnSubscribe( () -> {
-statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, 
jobId, StatusService.Status.STARTED,
-new HashMap<>() ).toBlocking().lastOrDefault( null );
-} ).doOnCompleted( () -> {
+ 
statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
+ StatusService.Status.INPROGRESS, 
status).toBlocking().lastOrDefault(null);
+ }).doOnSubscribe(() -> {
+
+statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+jobId, StatusService.Status.STARTED, new 
HashMap<>()).toBlocking().lastOrDefault(null);
+
+}).doOnCompleted(() -> {
 
 final long runningTotal = count.get();
 
 final Map status = new HashMap() {{
-put( "countProcessed", runningTotal );
-put( "updatedTimestamp", System.currentTimeMillis() );
+put("countProcessed", runningTotal);
+put("updatedTimestamp", System.currentTimeMillis());
 }};
 
-statusService
-.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, 
StatusService.Status.COMPLETE, status );
-} ).subscribeOn( Schedulers.newThread() ).subscribe();
+statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+jobId, StatusService.Status.COMPLETE, 
status).toBlocking().lastOrDefault(null);
+
+}).doOnError( (throwable) -> {
+logger.error("Error deduping connections", throwable);
+
+final Map status = new HashMap() {{
+put("error", throwable.getMessage() );
+}};
+
+statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+jobId, StatusService.Status.FAILED, 
status).toBlocking().lastOrDefault(null);;
+
+}

[3/3] usergrid git commit: Merge branch '2.1-release' into multitenant-migration

2015-10-29 Thread snoopdave 
Merge branch '2.1-release' into multitenant-migration


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b816a188
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b816a188
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b816a188

Branch: refs/heads/multitenant-migration
Commit: b816a188bade82ec1044e4dfb80220da67182b3c
Parents: 8af152e c652171
Author: Dave Johnson 
Authored: Thu Oct 29 20:24:14 2015 -0400
Committer: Dave Johnson 
Committed: Thu Oct 29 20:24:14 2015 -0400

--
 .../corepersistence/index/IndexServiceImpl.java |   2 +-
 .../persistence/index/CandidateResult.java  |  11 +-
 .../persistence/index/EntityIndexBatch.java |   2 +-
 .../usergrid/persistence/index/IndexFig.java|   6 +
 .../index/impl/DeIndexOperation.java|   5 +
 .../index/impl/EsEntityIndexBatchImpl.java  |  34 -
 .../index/impl/EsEntityIndexImpl.java   | 149 +++
 .../persistence/index/impl/IndexingUtils.java   |   2 +-
 .../persistence/index/impl/EntityIndexTest.java |  14 +-
 9 files changed, 143 insertions(+), 82 deletions(-)
--

[2/3] usergrid git commit: Fix failing test to ensure we don't set the same timestamp for all entities inserted into the index (since we don't do actually do that). Also ensure it's a long as what's r

2015-10-29 Thread snoopdave 
Fix failing test to ensure we don't set the same timestamp for all entities 
inserted into the index (since we don't do actually do that).  Also ensure it's 
a long as what's required of the timestamps


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c652171f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c652171f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c652171f

Branch: refs/heads/multitenant-migration
Commit: c652171f476f6ad953c94328e0fb2f4ea0b3aa44
Parents: 22beca2
Author: Michael Russo 
Authored: Wed Oct 28 16:56:19 2015 -0700
Committer: Michael Russo 
Committed: Wed Oct 28 16:56:19 2015 -0700

--
 .../persistence/index/impl/EntityIndexTest.java   | 14 ++
 1 file changed, 6 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c652171f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
--
diff --git 
a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
 
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 5243d5a..d6758dd 100644
--- 
a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ 
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import com.google.common.base.Optional;
 import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
 import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.model.field.*;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -48,10 +49,6 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.field.ArrayField;
-import org.apache.usergrid.persistence.model.field.IntegerField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
 import org.apache.usergrid.persistence.model.util.EntityUtils;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 
@@ -394,7 +391,6 @@ public class EntityIndexTest extends BaseIT {
 int numberOfEntities = 1000;
 int versionToSearchFor = numberOfEntities / 2;
 
-IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", 
SearchEdge.NodeType.SOURCE, 1 );
 
 UUID entityUUID = UUID.randomUUID();
 Id entityId = new SimpleId( "mehCar" );
@@ -408,9 +404,11 @@ public class EntityIndexTest extends BaseIT {
 Entity[] entity = new Entity[numberOfEntities];
 for(int i = 0; i < numberOfEntities; i++) {
 entity[i] = EntityIndexMapUtils.fromMap( entityMap );
-EntityUtils.setId( entity[i], entityId );
-EntityUtils.setVersion( entity[i], UUIDGenerator.newTimeUUID() );
-entity[i].setField( new UUIDField( 
IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID ) );
+EntityUtils.setId(entity[i], entityId);
+EntityUtils.setVersion(entity[i], UUIDGenerator.newTimeUUID());
+entity[i].setField(new 
UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID));
+
+IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", 
SearchEdge.NodeType.SOURCE, System.currentTimeMillis()*1000 );
 
 //index the new entity. This is where the loop will be set to 
create like 100 entities.
 indexProducer.put(entityIndex.createBatch().index( searchEdge, 
entity[i]  ).build()).subscribe();