Repository: usergrid Updated Branches: refs/heads/master 844468725 -> 79492c294
Add option to update each entity. Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/33fe480a Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/33fe480a Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/33fe480a Branch: refs/heads/master Commit: 33fe480a0c0585ac0a4861eda996cce6e84cce94 Parents: 8444687 Author: Dave Johnson <snoopd...@gmail.com> Authored: Tue Dec 13 09:11:49 2016 -0500 Committer: Dave Johnson <snoopd...@gmail.com> Committed: Tue Dec 13 09:11:49 2016 -0500 ---------------------------------------------------------------------- stack/tools/pom.xml | 10 ++++- .../org/apache/usergrid/tools/Repersist.java | 46 ++++++++++++++------ .../apache/usergrid/tools/RepersistTest.java | 6 +++ 3 files changed, 48 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/33fe480a/stack/tools/pom.xml ---------------------------------------------------------------------- diff --git a/stack/tools/pom.xml b/stack/tools/pom.xml index 3270683..817cbb6 100644 --- a/stack/tools/pom.xml +++ b/stack/tools/pom.xml @@ -54,14 +54,15 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> + <configuration> - <skipTests>false</skipTests> <systemPropertyVariables> <storage-config>${basedir}/src/test/conf</storage-config> </systemPropertyVariables> <forkMode>always</forkMode> <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine> </configuration> + </plugin> <plugin> @@ -260,5 +261,12 @@ <version>${aws.version}</version> </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>${mockito.version}</version> + <scope>test</scope> + </dependency> + </dependencies> </project> http://git-wip-us.apache.org/repos/asf/usergrid/blob/33fe480a/stack/tools/src/main/java/org/apache/usergrid/tools/Repersist.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Repersist.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Repersist.java index f16b2dc..9d023e3 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/Repersist.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Repersist.java @@ -18,6 +18,7 @@ package org.apache.usergrid.tools; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; @@ -31,17 +32,30 @@ import java.util.UUID; /** - * Iterate through all data in a list of apps and re-persist every entity in every collection. + * Iterate through all data in a list of apps and get each or update entity. */ public class Repersist extends ToolBase { static final Logger logger = LoggerFactory.getLogger( Repersist.class ); + int count = 0; + @Override @SuppressWarnings("static-access") public Options createOptions() { + Options options = super.createOptions(); - options.addOption("apps", true, "Comma-separated list of apps to re-persisted in {appName}/{orgName} format"); - options.addOption("wait (ms)", true, "Time to wait between entity re-persist"); + + Option appsOpton = new Option("apps", true, + "Comma-separated list of apps to re-persisted in {appName}/{orgName} format"); + appsOpton.setRequired(true); + options.addOption( appsOpton ); + + options.addOption("wait", true, + "Time in milliseconds to wait between entity re-persist"); + + options.addOption("update", false, + "Tool will update each and every entity in the listed apps"); + return options; } @@ -55,23 +69,26 @@ public class Repersist extends ToolBase { long wait = 100; try { - Long.parseLong(line.getOptionValue("wait")); + wait = Long.parseLong(line.getOptionValue("wait")); } catch( Exception e ) { logger.error("Incorrect or missing wait time, using default: " + wait); } + boolean update = line.hasOption("update"); + for ( String orgApp : orgApps ) { String[] orgAppParts = orgApp.split("/"); String org = orgAppParts[0]; String app = orgAppParts[1]; - repersist( org, org + "/" + app, wait ); + repersist( org, org + "/" + app, update, wait ); } } - private void repersist( String organizationName, String applicationName, long wait ) throws Exception { + private void repersist( String organizationName, String applicationName, boolean update, long wait ) + throws Exception { logger.info( "\n\nRepersisting {}/{}\n", organizationName, applicationName ); @@ -87,25 +104,29 @@ public class Repersist extends ToolBase { for ( String collectionName : collectionMetadata.keySet() ) { - int count = 0; - Query query = new Query(); query.setLimit( MAX_ENTITY_FETCH ); Results results = em.searchCollection( em.getApplicationRef(), collectionName, query ); while (results.size() > 0) { - for (Entity entity : results.getEntities()) { - // repersist - em.update(entity); + for (Entity entity : results.getEntities()) { + if ( update ) { + // TODO: can we do this without updating the update of each entity? + em.update(entity); + } count++; + if ( count % 1000 == 0 ) { + logger.info("Processed {} entities", count); + } Thread.sleep( wait ); - } + if (results.getCursor() == null) { break; } + query.setCursor( results.getCursor() ); results = em.searchCollection( em.getApplicationRef(), collectionName, query ); } @@ -114,7 +135,6 @@ public class Repersist extends ToolBase { organizationName, applicationName, collectionName, count); } - } } http://git-wip-us.apache.org/repos/asf/usergrid/blob/33fe480a/stack/tools/src/test/java/org/apache/usergrid/tools/RepersistTest.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/RepersistTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/RepersistTest.java index 4d65c16..348799b 100644 --- a/stack/tools/src/test/java/org/apache/usergrid/tools/RepersistTest.java +++ b/stack/tools/src/test/java/org/apache/usergrid/tools/RepersistTest.java @@ -24,6 +24,8 @@ import org.junit.ClassRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.junit.Assert.assertEquals; + public class RepersistTest extends AbstractServiceIT { static final Logger logger = LoggerFactory.getLogger( RepersistTest.class ); @@ -48,6 +50,8 @@ public class RepersistTest extends AbstractServiceIT { "-organization", orgName, "-application", appName, "-username", userName, + "-users", "5", + "-collections", "1", "-host", "localhost:9160", "-eshost", "localhost:9200", "-escluster", "elasticsearch", @@ -67,6 +71,8 @@ public class RepersistTest extends AbstractServiceIT { }, false); + assertEquals( 38, repersist.count ); + } }
