Author: challngr Date: Wed Oct 28 18:12:53 2015 New Revision: 1711088 URL: http://svn.apache.org/viewvc?rev=1711088&view=rev Log: UIMA-4577 Switch from OrientDb to Cassandra.
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbVerify.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ReadCkpt.java Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Wed Oct 28 18:12:53 2015 @@ -115,6 +115,9 @@ class Ducc(DuccUtil): def add_to_classpath(self, lib): os.environ['CLASSPATH'] = os.environ['CLASSPATH'] + ":" + lib + def prepend_classpath(self, lib): + os.environ['CLASSPATH'] = lib + ":" + os.environ['CLASSPATH'] + def run_component(self, component, or_parms, numagents, rmoverride, background, nodup, localdate): if ( component == 'all' ): @@ -181,7 +184,7 @@ class Ducc(DuccUtil): jvm_opts.append("-Dducc.rm.override.dram=" + rmoverride) if ( self.rm_jvm_args != None ): jvm_opts.append(self.rm_jvm_args) - self.add_to_classpath(ducc_home + '/lib/orientdb/*') + self.prepend_classpath(ducc_home + '/lib/cassandra/*') if ( c == 'ws' ): #see if the ws jsp compilation directory is specified @@ -200,7 +203,7 @@ class Ducc(DuccUtil): self.add_to_classpath(ducc_home + '/lib/http-client/*') self.add_to_classpath(ducc_home + '/webserver/lib/*') self.add_to_classpath(ducc_home + '/webserver/lib/jsp/*') - self.add_to_classpath(ducc_home + '/lib/orientdb/*') + self.prepend_classpath(ducc_home + '/lib/cassandra/*') if ( c == 'orchestrator' ): if ( or_parms != None ): @@ -209,7 +212,7 @@ class Ducc(DuccUtil): jvm_opts.append(self.or_jvm_args) self.add_to_classpath(ducc_home + '/lib/http-client/*') self.add_to_classpath(ducc_home + '/webserver/lib/*') - self.add_to_classpath(ducc_home + '/lib/orientdb/*') + self.prepend_classpath(ducc_home + '/lib/cassandra/*') if ( c == 'pm' ): if ( self.pm_jvm_args != None ): @@ -221,7 +224,7 @@ class Ducc(DuccUtil): self.add_to_classpath(ducc_home + '/apache-uima/apache-activemq/lib/optional/*') self.add_to_classpath(ducc_home + '/lib/http-client/*') self.add_to_classpath(ducc_home + '/webserver/lib/*') - self.add_to_classpath(ducc_home + '/lib/orientdb/*') + self.prepend_classpath(ducc_home + '/lib/cassandra/*') if (component != 'agent'): service = 'org.apache.uima.ducc.common.main.DuccService' Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Wed Oct 28 18:12:53 2015 @@ -239,6 +239,7 @@ class DuccUtil(DuccBase): def db_start(self): + return True # bypass all of this for the initial delivery if ( self.db_parms == self.db_disabled ): print ' (Bypass database start because ducc.database.host =', self.db_disabled + ')' @@ -288,7 +289,7 @@ class DuccUtil(DuccBase): return False def db_init(self): - + return True # bypass all of this for the initial delivery if ( self.db_parms == self.db_disabled ): return True @@ -304,6 +305,7 @@ class DuccUtil(DuccBase): self.spawn(cmd) def db_stop(self): + return True # bypass all of this for the initial delivery if ( self.db_parms == self.db_disabled ): return True Modified: uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml (original) +++ uima/sandbox/uima-ducc/trunk/src/main/assembly/bin.xml Wed Oct 28 18:12:53 2015 @@ -130,80 +130,29 @@ under the License. <directoryMode>755</directoryMode> </dependencySet> - <!-- orientdb --> - <dependencySet> - <useStrictFiltering>true</useStrictFiltering> - <includes> - <include>com.tinkerpop.gremlin:gremlin-groovy</include> - </includes> - <outputDirectory>lib/orientdb</outputDirectory> - <useProjectArtifact>false</useProjectArtifact> - <fileMode>644</fileMode> - <directoryMode>755</directoryMode> - </dependencySet> - - <dependencySet> - <useStrictFiltering>true</useStrictFiltering> - <includes> - <include>com.tinkerpop.blueprints:blueprints-core</include> - </includes> - <outputDirectory>lib/orientdb</outputDirectory> - <useProjectArtifact>false</useProjectArtifact> - <fileMode>644</fileMode> - <directoryMode>755</directoryMode> - </dependencySet> <dependencySet> <useStrictFiltering>true</useStrictFiltering> <includes> - <include>com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru</include> + <include>com.datastax.cassandra:cassandra-driver-core</include> + <include>com.datastax.cassandra:cassandra-driver-dse</include> + <include>com.datastax.cassandra:cassandra-driver-mapping</include> + <include>io.netty:netty-buffer</include> + <include>io.netty:netty-codec</include> + <include>io.netty:netty-common</include> + <include>io.netty:netty-handler</include> + <include>io.netty:netty-transport</include> + <include>com.codahale.metrics:metrics-core</include> + <include>org.xerial.snappy:snappy-java</include> + <include>net.jpountz.lz4:lz4</include> + <include>org.hdrhistogram:HdrHistogram</include> </includes> - <outputDirectory>lib/orientdb</outputDirectory> + <outputDirectory>lib/cassandra</outputDirectory> <useProjectArtifact>false</useProjectArtifact> <fileMode>644</fileMode> <directoryMode>755</directoryMode> </dependencySet> - <dependencySet> - <useStrictFiltering>true</useStrictFiltering> - <includes> - <include>com.orientechnologies:orientdb-graphdb</include> - <include>com.orientechnologies:orientdb-core</include> - <include>com.orientechnologies:orientdb-client</include> - <include>com.orientechnologies:orientdb-server</include> - <include>com.orientechnologies:orientdb-tools</include> - <include>com.orientechnologies:orientdb-enterprise</include> - </includes> - <outputDirectory>lib/orientdb</outputDirectory> - <useProjectArtifact>false</useProjectArtifact> - <fileMode>644</fileMode> - <directoryMode>755</directoryMode> - </dependencySet> - - <dependencySet> - <useStrictFiltering>true</useStrictFiltering> - <includes> - <include>com.tinkerpop.gremlin:gremlin-groovy</include> - </includes> - <outputDirectory>lib/orientdb</outputDirectory> - <useProjectArtifact>false</useProjectArtifact> - <fileMode>644</fileMode> - <directoryMode>755</directoryMode> - </dependencySet> - - <!-- dependencySet> - <useStrictFiltering>true</useStrictFiltering> - <includes> - <include>com.orientechnologies:orientdb-lucene</include> - </includes> - <outputDirectory>database/plugins</outputDirectory> - <useProjectArtifact>false</useProjectArtifact> - <fileMode>644</fileMode> - <directoryMode>755</directoryMode> - </dependencySet --> - - <!-- end orientdb --> - <!-- xmlbeans is in UIMA-AS 2.6.0 package --> <dependencySet> Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties (original) +++ uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties Wed Oct 28 18:12:53 2015 @@ -510,7 +510,7 @@ ducc.process.thread.sleep.time=3000 ducc.flow-controller.specifier=org.apache.uima.ducc.FlowController -# database things ----------------------------------------------------------------------------------- +# database # Directory where the databases and associated stuff is kept, relative to DUCC_HOME. This # is ORIENTDB_HOME ducc.database.location = database @@ -520,16 +520,8 @@ ducc.database.config.file = resource ducc.database.host = --disabled-- -# The binary url not HTTP/REST -ducc.database.url = remote:${ducc.database.host}:2424 -# The rest interface -ducc.database.rest.url = ${ducc.database.host}:2480 - # Name of the database with active state. Svc registry, etc. -ducc.state.database.url = ${ducc.database.url}/DuccState - -# Name of the database with history -ducc.history.database.url = ${ducc.database.url}/DuccHistory +ducc.state.database.url = ${ducc.head} # This class implementes the persistence interface for the services registry #ducc.service.persistence.impl = org.apache.uima.ducc.database.StateServicesDb Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java?rev=1711088&view=auto ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java (added) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IDbProperty.java Wed Oct 28 18:12:53 2015 @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. +*/ + +package org.apache.uima.ducc.common.persistence; + +/** + * All properties sent to the DB must implement this interface. + */ +public interface IDbProperty +{ + String pname(); // This is the name of the property. Any java.util.Property key .toString() is valid. + Type type(); // One of the types below. DB impl translates to db equialent. Try to keep these + // an easy analogy of java names. DB does not convert so the type of the incoming + // propety has to match. + boolean isPrimaryKey(); // Primary key in DB. Many columns can have this which results in a compount key. Order + // is important - first is cluster key, and following are the others. + boolean isPrivate(); // Used only in the DB, do not pass back to the application in Properties. This goes into the db and + // is part of the schema. + boolean isMeta(); // A handy constant in the enum that isn't part of a schema. Helps in the generation + // of the schema. Doesn't go into the DB. + String columnName(); // Because we want to persist java.util.Properties, and these things + // allow a greater range of characters in the keys ('.', '-' and others) + // than does SQL, we define a translation from the "properties" key to + // legal SQL syntactic names. DB does not translate, user must provide + // a suitable translation. + + // If we update this we may have to update db methods that use it + public enum Type { + String, // Java String + Blob, // Java serialized object or other binary + Boolean, // Java boolean + Integer, + Long, + Double, + UUID, + }; +} Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java Wed Oct 28 18:12:53 2015 @@ -22,6 +22,7 @@ package org.apache.uima.ducc.common.pers import java.util.Map; import java.util.Properties; +import org.apache.uima.ducc.common.persistence.IDbProperty; import org.apache.uima.ducc.common.utils.DuccLogger; public interface IRmPersistence @@ -50,7 +51,7 @@ public interface IRmPersistence * @param id This is the primary key, the machine name; * @param properties These are the props, must be presented in the form of (String, Object) ... */ - public void setProperties(Object dbid, String id, Object... properties) throws Exception; + public void setProperties(String id, Object... properties) throws Exception; /** * Set a property on an object. If the property cannot be set the action @@ -64,7 +65,7 @@ public interface IRmPersistence * throw will originate in the DB because of some DB issue. An * exception causes the action to be rolled back. */ - public void setProperty(Object dbid, String id, RmPropName key, Object value) throws Exception; + public void setProperty(String id, RmProperty key, Object value) throws Exception; /** @@ -82,7 +83,7 @@ public interface IRmPersistence * * @return The db id of the created machine. */ - public Object createMachine(String id, Properties props) throws Exception; + public void createMachine(String id, Map<RmProperty, Object> props) throws Exception; /** * Fetch a machine by its id. @@ -110,58 +111,74 @@ public interface IRmPersistence */ public Map<String, Properties> getAllMachines() throws Exception; - public String toGson(Object o); - public interface RmProps - { - String pname(); - } - - enum RmPropName - implements RmProps + enum RmProperty + implements IDbProperty { + TABLE_NAME { + public String pname() { return "rmnodes"; } + public Type type() { return Type.String; } + public boolean isPrivate() { return true;} + public boolean isMeta() { return true;} + }, Name { public String pname() { return "name"; } + public Type type() { return Type.String; } + public boolean isPrimaryKey() { return true;} + }, Responsive{ public String pname() { return "responsive"; } + public Type type() { return Type.Boolean; } }, Online{ public String pname() { return "online"; } - }, - HeartBeats { - public String pname() { return "heartbeats"; } + public Type type() { return Type.Boolean; } }, Ip { public String pname() { return "ip"; } + public Type type() { return Type.String; } }, Nodepool { public String pname() { return "nodepool"; } + public Type type() { return Type.String; } }, Quantum { public String pname() { return "quantum"; } + public Type type() { return Type.Integer; } }, Memory { public String pname() { return "memory"; } + public Type type() { return Type.Integer; } }, ShareOrder { public String pname() { return "share_order"; } + public Type type() { return Type.Integer; } }, Shares{ public String pname() { return "shares"; } + public Type type() { return Type.Integer; } }, Blacklisted { public String pname() { return "blacklisted"; } + public Type type() { return Type.Boolean; } }, Heartbeats { public String pname() { return "heartbeats"; } + public Type type() { return Type.Integer; } }, SharesLeft { public String pname() { return "shares_left"; } + public Type type() { return Type.Integer; } }, Assignments { public String pname() { return "assignments"; } + public Type type() { return Type.Integer; } }, ; + public boolean isPrimaryKey() { return false; } + public boolean isPrivate() { return false; } + public boolean isMeta() { return false; } + public String columnName() { return pname(); } } } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java Wed Oct 28 18:12:53 2015 @@ -33,11 +33,9 @@ public class NullRmStatePersistence impl public void init(DuccLogger logger) throws Exception { } public void clear() {} - public void setProperty(Object dbid, String id, RmPropName key, Object value) { } - public void setProperties(Object dbid, String id, Object... props) {} - public Object createMachine(String id, Properties props) { return new Integer(1);} + public void setProperty(String id, RmProperty key, Object value) { } + public void setProperties(String id, Object... props) {} + public void createMachine(String id, Map<RmProperty, Object> props) { } public Properties getMachine(String id) { return null; } public Map<String, Properties> getAllMachines() { return new HashMap<String, Properties>(); } - public String toGson(Object o) { return ""; } - } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/RmPersistenceFactory.java Wed Oct 28 18:12:53 2015 @@ -59,7 +59,7 @@ public class RmPersistenceFactory ret = (IRmPersistence) iss.newInstance(); ret.init(logger); } catch ( Throwable t ) { - logger.error(methodName, null, "Cannot instantiate RM persistence class", clname, ":", t); + logger.error(methodName, null, "Cannot instantiate RM persistence class", clname, ":", t, "Using NullRmStatePersistance as default."); ret = new NullRmStatePersistence(); } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/IStateServices.java Wed Oct 28 18:12:53 2015 @@ -18,10 +18,10 @@ */ package org.apache.uima.ducc.common.persistence.services; -import java.util.List; import java.util.Properties; import org.apache.uima.ducc.common.IDuccEnv; +import org.apache.uima.ducc.common.persistence.IDbProperty; import org.apache.uima.ducc.common.utils.DuccLogger; import org.apache.uima.ducc.common.utils.id.DuccId; @@ -31,104 +31,231 @@ public interface IStateServices { public static String svc_reg_dir = IDuccEnv.DUCC_STATE_SVCREG_DIR; public static String svc_hist_dir = IDuccEnv.DUCC_HISTORY_SVCREG_DIR; - - public static final String archive_key = "is_archived"; - public static final String archive_flag = "true"; - public static final String svc = "svc"; public static final String meta = "meta"; - public enum SvcProps { - endpoint { public String pname() { return "endpoint"; } }, - instances { public String pname() { return "instances"; } }, - autostart { public String pname() { return "autostart"; } }, - reference { public String pname() { return "reference"; } }, - enabled { public String pname() { return "enabled"; } }, + public static final String UIMA_AS = "UIMA-AS"; + public static final String CUSTOM = "CUSTOM"; + + public static final String sequenceKey = "service.seqno"; - disable_reason { public String pname() { return "disable_reason"; } }, + // IMPORTANT IMPORTANT IMPORANT + // For DB: we must distinguish between properties for the registration itself, and + // metaprops. + // IMPORTANT IMPORTANT IMPORANT + + // All this will be strings I guess, because they're all entered by humans + // and all the code is already debugged to deal with conversion. + public enum SvcRegProps + implements IDbProperty + { + TABLE_NAME { + public String pname() { return "smreg"; } + public Type type() { return Type.String; } + public boolean isPrivate() { return true; } + public boolean isMeta() { return true; } + }, + + numeric_id { + public String pname() { return "numeric_id"; } + public boolean isPrimaryKey() { return true; } + public boolean isPrivate() { return true; } + + }, + + uuid { + public String pname() { return "uuid"; } + public boolean isPrivate() { return true; } + }, + + is_archived { + public String pname() { return "is_archived"; } + public Type type() { return Type.Boolean; } + public boolean isPrivate() { return true; } + }, + + description { public String pname() { return "description"; } }, + administrators { public String pname() { return "administrators"; } }, + scheduling_class { public String pname() { return "scheduling_class"; } }, + log_directory { public String pname() { return "log_directory"; } }, + working_directory { public String pname() { return "working_directory"; } }, + jvm { public String pname() { return "jvm"; } }, + process_jvm_args { public String pname() { return "process_jvm_args"; } }, + classpath { public String pname() { return "classpath"; } }, + environment { public String pname() { return "environment"; } }, + + process_memory_size { public String pname() { return "process_memory_size"; } }, + process_dd { public String pname() { return "process_descriptor_DD"; } }, + process_debug { public String pname() { return "process_debug"; } }, + process_executable { public String pname() { return "process_executable"; } }, + process_executable_args { public String pname() { return "process_executable_args"; } }, + process_initialization_time_max { public String pname() { return "process_initialization_time_max"; } }, + + service_dependency { public String pname() { return "service_dependency"; } }, + service_request_endpoint { public String pname() { return "service_request_endpoint"; } }, + service_linger { public String pname() { return "service_linger"; } }, + + service_ping_arguments { public String pname() { return "service_ping_arguments"; } }, + service_ping_class { public String pname() { return "service_ping_class"; } }, + service_ping_classpath { public String pname() { return "service_ping_classpath"; } }, + service_ping_jvm_args { public String pname() { return "service_ping_jvm_args"; } }, + service_ping_dolog { public String pname() { return "service_ping_dolog"; } }, + service_ping_timeout { public String pname() { return "service_ping_timeout"; } }, + + instance_failures_window { public String pname() { return "instance_failures_window"; } }, + instance_failures_limit { public String pname() { return "instance_failures_limit"; } }, + instance_init_failures_limit { public String pname() { return "instance_init_failures_limit"; } }, + + ; + public Type type() { return Type.String; } + public boolean isPrimaryKey() { return false; } + public boolean isPrivate() { return false; } + public boolean isMeta() { return false; } + public String columnName() { return pname(); } + }; + + // These will be strings as well because there's code all over assuming they're strings. + public enum SvcMetaProps + implements IDbProperty + { + TABLE_NAME { + public String pname() { return "smmeta"; } + public Type type() { return Type.String; } + public boolean isPrivate() { return true; } + public boolean isMeta() { return true; } + }, + + numeric_id { + public String pname() { return "numeric_id"; } + public boolean isPrimaryKey() { return true; } + }, + + is_archived { + public String pname() { return "is_archived"; } + public Type type() { return Type.Boolean; } + public boolean isPrivate() { return true; } + }, - implementors { public String pname() { return "implementors"; } }, - numeric_id { public String pname() { return "numeric_id"; } }, uuid { public String pname() { return "uuid"; } }, - service_seqno { public String pname() { return "service.seqno"; } }, - ping_active { public String pname() { return "ping_active"; } }, - ping_only { public String pname() { return "ping_only"; } }, - - service_alive { public String pname() { return "service_alive"; } }, - service_class { public String pname() { return "service_class"; } }, - service_dependency { public String pname() { return "service_dependency"; } }, - service_healthy { public String pname() { return "service_healthy"; } }, - service_state { public String pname() { return "service_state"; } }, - last_use { public String pname() { return "last_use"; } }, - last_use_readable { public String pname() { return "last_use_readable"; } }, - service_statistics { public String pname() { return "service_statistics"; } }, - service_type { public String pname() { return "service_type"; } }, - submit_error { public String pname() { return "submit_error"; } }, - user { public String pname() { return "user"; } }, - references { public String pname() { return "references"; } }, - stopped { public String pname() { return "stopped"; } }, - - scheduling_class { public String pname() { return "scheduling_class"; } }, - process_memory_size { public String pname() { return "process_memory_size"; } }, - description { public String pname() { return "description"; } }, - log_directory { public String pname() { return "log_directory"; } }, - process_executable { public String pname() { return "process_executable"; } }, - - last_ping { public String pname() { return "last_ping"; } }, - last_ping_readable { public String pname() { return "last_ping_readable"; } }, - last_runnable { public String pname() { return "last_runnable"; } }, - last_runnable_readable { public String pname() { return "last_runnable_readable"; } }, - work_instances { public String pname() { return "work_instances"; } }, - registration_date { public String pname() { return "registration_date"; } }, - registration_date_millis { public String pname() { return "registration_date_millis"; } }, + reference { public String pname() { return "reference"; } }, // boolean, is this reference s + enabled { public String pname() { return "enabled"; } }, + autostart { public String pname() { return "autostart"; } }, // here we specify whether it is autostarted + instances { public String pname() { return "instances"; } }, + endpoint { public String pname() { return "endpoint"; } }, - instance_init_failures_limit { public String pname() { return "instance_init_failures_limit"; } }, + disable_reason { + public String pname() { return "disable-reason"; } + public String columnName() { return "disable_reason"; } + }, + + implementors { + public String pname() { return "implementors"; } + }, + + ping_active { + public String pname() { return "ping-active"; } + public String columnName() { return "ping_active"; } + }, + ping_only { + public String pname() { return "ping-only"; } + public String columnName() { return "ping_only"; } + }, + + service_alive { + public String pname() { return "service-alive"; } + public String columnName() { return "service_alive"; } + }, + service_class { + public String pname() { return "service-class"; } + public String columnName() { return "service_class"; } + }, + service_dependency { + public String pname() { return "service-dependency"; } + public String columnName() { return "service_dependency"; } + }, + service_healthy { + public String pname() { return "service-healthy"; } + public String columnName() { return "service_healthy"; } + }, + service_state { + public String pname() { return "service-state"; } + public String columnName() { return "service_state"; } + }, + last_use { + public String pname() { return "last-use"; } + public String columnName() { return "last_use"; } + }, + last_use_readable { + public String pname() { return "last-use-readable"; } + public String columnName() { return "last_use_readable"; } + }, + service_statistics { + public String pname() { return "service-statistics"; } + public String columnName() { return "service_statistics"; } + }, + service_type { + public String pname() { return "service-type"; } + public String columnName() { return "service_type"; } + }, + submit_error { + public String pname() { return "submit-error"; } + public String columnName() { return "submit_error"; } + }, + user { + public String pname() { return "user"; } + }, - UIMA_AS { public String pname() { return "UIMA-AS"; } }, - CUSTOM { public String pname() { return "CUSTOM"; } }, + references { public String pname() { return "references"; } }, // things that reference me + stopped { public String pname() { return "stopped"; } }, + + last_ping { + public String pname() { return "last-ping"; } + public String columnName() { return "last_ping"; } + }, + last_ping_readable { + public String pname() { return "last-ping-readable"; } + public String columnName() { return "last_ping_readable"; } + }, + last_runnable { + public String pname() { return "last-runnable"; } + public String columnName() { return "last_runnable"; } + }, + last_runnable_readable { + public String pname() { return "last-runnable-readable"; } + public String columnName() { return "last_runnable_readable"; } + }, + work_instances { + public String pname() { return "work-instances"; } + public String columnName() { return "work_instances"; } + }, + registration_date { + public String pname() { return "registration-date"; } + public String columnName() { return "registration_date"; } + }, + registration_date_millis { + public String pname() { return "registration-date-millis"; } + public String columnName() { return "registration_date_millis"; } + }, ; - public abstract String pname(); - }; + public Type type() { return Type.String; } + public boolean isPrimaryKey() { return false; } + public boolean isPrivate() { return false; } + public boolean isMeta() { return false; } + public String columnName() { return pname(); } - //public static final String endpoint { public String pname() { return "endpoint"; } }, - //public static final String instances = "instances"; - //public static final String autostart = "autostart"; - - //public static final String reference = "reference"; - //public static final String enabled = "enabled"; - //public static final String disable_reason = "disable-reason"; - //public static final String implementors = "implementors"; - //public static final String numeric_id = "numeric_id"; - //public static final String ping_active = "ping-active"; - //public static final String ping_only = "ping-only"; - //public static final String service_alive = "service-alive"; - //public static final String service_class = "service-class"; - //public static final String service_dependency = "service_dependency"; - //public static final String service_healthy = "service-healthy"; - //public static final String service_state = "service-state"; - //public static final String last_use = "last-use"; - //public static final String service_statistics = "service-statistics"; - //public static final String service_type = "service-type"; - //public static final String submit_error = "submit-error"; - //public static final String user = "user"; - - //public static final String scheduling_class = "scheduling_class"; - //public static final String process_memory_size = "process_memory_size"; - //public static final String description = "description"; - //public static final String log_directory = "log_directory"; - //public static final String process_executable = "process_executable"; - - //public static final String service_type_UIMA_AS = "UIMA-AS"; - //public static final String service_type_CUSTOM = "CUSTOM"; + + }; // - // IMPORTANT IMPORTANT IMPORTANT - // As of now, 2015/08/17, there is no support for service registration history. There never has been in - // this class and it's derivatives. Until there is a use-case and demand it's not clear what the interfaces - // would be like. + // IMPORTANT IMPORTANT IMPORTANT + // + // As of now, 2015/08/17, there is no support for service registration history in this class. + // There never has been direct DUCC support for this other than keeping the old registrations + // around. Until there is a use-case and demand it's not clear what the interfaces would be + // like. // // We may have to prune, or look at specific history files, but that can be done with direct database // operations. If we need more, well add it later. @@ -137,15 +264,16 @@ public interface IStateServices { // 'history' state. // IMPORTANT IMPORTANT IMPORTANT - public List<Long> getSvcList() throws Exception; // list of registered services - public List<Long> getMetaList() throws Exception; // not used ? + // Never used except in test. Removing them pending complaints + //public List<Long> getSvcList() throws Exception; // list of registered services + //public List<Long> getMetaList() throws Exception; // not used ? public StateServicesDirectory getStateServicesDirectory() throws Exception; // all the registy in one blow public boolean storeProperties (DuccId serviceId, Properties svc, Properties meta) throws Exception; // save svc and meta in a transaction - public boolean updateJobProperties (Object dbid, DuccId serviceId, Properties props) throws Exception; // update just job props - public boolean updateMetaProperties(Object dbid, DuccId serviceId, Properties props) throws Exception; // update just metaprops - public void moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception; + public boolean updateJobProperties (DuccId serviceId, Properties props) throws Exception; // update just job props + public boolean updateMetaProperties(DuccId serviceId, Properties props) throws Exception; // update just metaprops + public boolean moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception; public void shutdown() throws Exception; public boolean init(DuccLogger logger) throws Exception; Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/NullStateServices.java Wed Oct 28 18:12:53 2015 @@ -19,7 +19,6 @@ package org.apache.uima.ducc.common.persistence.services; import java.io.IOException; -import java.util.ArrayList; import java.util.Properties; import org.apache.uima.ducc.common.utils.DuccLogger; @@ -33,33 +32,19 @@ public class NullStateServices implement } public boolean init(DuccLogger logger) throws Exception { return true; } - public ArrayList<Long> getSvcList() - throws Exception - { - return new ArrayList<Long>(); - } - - - public ArrayList<Long> getMetaList() - throws Exception - { - return new ArrayList<Long>(); - } - - + public StateServicesDirectory getStateServicesDirectory() throws IOException { return new StateServicesDirectory(); } - public boolean storeProperties(DuccId serviceId, Properties svc, Properties meta) throws Exception {return false;} - public boolean updateProperties(Object dbid, DuccId serviceId, String type, Properties props) throws Exception {return false;} + public boolean storeProperties(DuccId serviceId, Properties svc, Properties meta) throws Exception {return true;} public void deleteProperties(DuccId serviceId) throws Exception {} public void shutdown() throws Exception {} public void moveToHHistory() throws Exception {} - public boolean updateJobProperties(Object dbid, DuccId serviceId, Properties props) throws Exception {return false;} - public boolean updateMetaProperties(Object dbid, DuccId serviceId, Properties props) throws Exception {return false;} - public void moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception {} + public boolean updateJobProperties(DuccId serviceId, Properties props) throws Exception {return true;} + public boolean updateMetaProperties(DuccId serviceId, Properties props) throws Exception {return true;} + public boolean moveToHistory(DuccId serviceId, Properties svc, Properties meta) throws Exception {return true;} } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServices.java Wed Oct 28 18:12:53 2015 @@ -89,14 +89,14 @@ public class StateServices implements IS } - public ArrayList<Long> getSvcList() { + private ArrayList<Long> getSvcList() { return getList(IStateServices.svc); } - public ArrayList<Long> getMetaList() { - return getList(IStateServices.meta); - } +// private ArrayList<Long> getMetaList() { +// return getList(IStateServices.meta); +// } private DuccProperties getProperties(String name) { String location = "getProperties"; @@ -237,21 +237,21 @@ public class StateServices implements IS return ok; } - private boolean updateProperties(Object dbid, DuccId serviceId, Properties props, String type) + private boolean updateProperties(DuccId serviceId, Properties props, String type) { File f = new File(mkfilename(serviceId, type)); File tmpf = new File(f.toString() + ".tmp"); return saveProperties(serviceId, props, f, tmpf, type); } - public boolean updateJobProperties(Object dbid, DuccId serviceId, Properties props) + public boolean updateJobProperties(DuccId serviceId, Properties props) { - return updateProperties(dbid, serviceId, props, svc); + return updateProperties(serviceId, props, svc); } - public boolean updateMetaProperties(Object dbid, DuccId serviceId, Properties props) + public boolean updateMetaProperties(DuccId serviceId, Properties props) { - return updateProperties(dbid, serviceId, props, meta); + return updateProperties(serviceId, props, meta); } public void deleteProperties(long serviceId) @@ -272,11 +272,12 @@ public class StateServices implements IS } - public void moveToHistory(DuccId id, Properties svc, Properties meta) + public boolean moveToHistory(DuccId id, Properties svc, Properties meta) throws Exception { String methodName = "moveToHistory"; + boolean ret = true; // Save a copy in history, and then delete the original File mfh = new File(svc_hist_dir + id + ".meta"); try { @@ -300,12 +301,15 @@ public class StateServices implements IS fos.close(); } catch (Exception e) { logger.warn(methodName, null, id + ":Unable to save history to \"" + pfh.toString(), ": ", e.toString() + "\""); + ret = false; } String props_filename = svc_reg_dir + id + ".svc"; File pf = new File(props_filename); pf.delete(); + return ret; } public void shutdown() {} } + Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/services/StateServicesDirectory.java Wed Oct 28 18:12:53 2015 @@ -18,6 +18,7 @@ */ package org.apache.uima.ducc.common.persistence.services; +import java.util.Map; import java.util.NavigableSet; import java.util.TreeMap; @@ -36,4 +37,8 @@ public class StateServicesDirectory { public NavigableSet<Long> getDescendingKeySet() { return map.descendingKeySet(); } + + public Map<Long, StateServicesSet> getMap() { + return map; + } } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/test/java/org/apache/uima/ducc/common/test/cmd/StateServicesTest.java Wed Oct 28 18:12:53 2015 @@ -19,7 +19,6 @@ package org.apache.uima.ducc.common.test.cmd; import java.util.Iterator; -import java.util.List; import java.util.Map.Entry; import java.util.NavigableSet; import java.util.Properties; @@ -45,14 +44,16 @@ public class StateServicesTest return; } IStateServices ss = StateServicesFactory.getInstance(StateServicesTest.class.getName(), "TEST"); - List<Long> svcList = ss.getSvcList(); - for(Long fname : svcList) { - System.out.println(fname); - } - List<Long> metaList = ss.getMetaList(); - for(Long fname : metaList) { - System.out.println(fname); - } + + // These two routines are never used except in this test! Removing them pending complaints. +// List<Long> svcList = ss.getSvcList(); +// for(Long fname : svcList) { +// System.out.println(fname); +// } +// List<Long> metaList = ss.getMetaList(); +// for(Long fname : metaList) { +// System.out.println(fname); +// } StateServicesDirectory ssd = ss.getStateServicesDirectory(); NavigableSet<Long> keySet = ssd.getDescendingKeySet(); Iterator<Long> iterator = keySet.iterator(); Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/pom.xml Wed Oct 28 18:12:53 2015 @@ -63,36 +63,39 @@ <artifactId>uima-ducc-transport</artifactId> </dependency> + <dependency> + <groupId>com.datastax.cassandra</groupId> + <artifactId>cassandra-driver-core</artifactId> + <version>${cassandra.driver.version}</version> + </dependency> <dependency> - <groupId>com.orientechnologies</groupId> - <artifactId>orientdb-graphdb</artifactId> - <version>${orientdb.version}</version> + <groupId>com.datastax.cassandra</groupId> + <artifactId>cassandra-driver-dse</artifactId> + <version>${cassandra.driver.version}</version> </dependency> - + <dependency> - <groupId>com.orientechnologies</groupId> - <artifactId>orientdb-lucene</artifactId> - <version>${orientdb.version}</version> + <groupId>com.datastax.cassandra</groupId> + <artifactId>cassandra-driver-mapping</artifactId> + <version>${cassandra.driver.version}</version> </dependency> - + <dependency> - <groupId>com.tinkerpop.blueprints</groupId> - <artifactId>blueprints-core</artifactId> - <version>2.6.0</version> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> </dependency> <dependency> - <groupId>com.googlecode.concurrentlinkedhashmap</groupId> - <artifactId>concurrentlinkedhashmap-lru</artifactId> - <version>1.4.2</version> - </dependency> + <groupId>net.jpountz.lz4</groupId> + <artifactId>lz4</artifactId> + </dependency> <dependency> - <groupId>com.tinkerpop.gremlin</groupId> - <artifactId>gremlin-groovy</artifactId> - <version>2.6.0</version> + <groupId>org.hdrhistogram</groupId> + <artifactId>HdrHistogram</artifactId> </dependency> + </dependencies> </project> Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java?rev=1711088&r1=1711087&r2=1711088&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java Wed Oct 28 18:12:53 2015 @@ -19,18 +19,15 @@ package org.apache.uima.ducc.database; +import java.util.List; + import org.apache.uima.ducc.common.utils.DuccLogger; -import org.apache.uima.ducc.database.DbConstants.DbEdge; -import org.apache.uima.ducc.database.DbConstants.DbVertex; -import org.apache.uima.ducc.database.DbConstants.Index; - -import com.orientechnologies.orient.client.remote.OServerAdmin; -import com.orientechnologies.orient.core.metadata.schema.OProperty; -import com.orientechnologies.orient.core.sql.OCommandSQL; -import com.tinkerpop.blueprints.impls.orient.OrientEdgeType; -import com.tinkerpop.blueprints.impls.orient.OrientGraphFactory; -import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; -import com.tinkerpop.blueprints.impls.orient.OrientVertexType; + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.Host; +import com.datastax.driver.core.Metadata; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.SimpleStatement; public class DbCreate { @@ -38,8 +35,9 @@ public class DbCreate String dburl; String adminid = "root"; String adminpw = null; - OServerAdmin admin; - OrientGraphFactory factory; + + private Cluster cluster; + private Session session; public DbCreate(String dburl) { @@ -60,6 +58,31 @@ public class DbCreate this.adminpw = adminpw; } + public void connect() + { + String methodName = "connect"; + cluster = Cluster.builder() + .addContactPoint(dburl) + .build(); + + Metadata metadata = cluster.getMetadata(); + doLog(methodName, "Connected to cluster: %s\n", metadata.getClusterName()); + + for ( Host host : metadata.getAllHosts() ) { + doLog(methodName, "Datatacenter: %s; Host: %s; Rack: %s\n", host.getDatacenter(), host.getAddress(), host.getRack()); + } + session = cluster.connect(); + } + + public void close() { + cluster.close(); + } + + public Session getSession() + { + return this.session; + } + public void doLog(String methodName, Object ... msg) { if ( logger == null ) { @@ -72,8 +95,7 @@ public class DbCreate } else { buf.append(o.toString()); } - } - + } System.out.println(buf); } else { logger.info(methodName, null, msg); @@ -82,162 +104,74 @@ public class DbCreate } - void createEdgeType(OrientGraphNoTx g, DbEdge id) + String mkTableCreate(String tableName, String[] fields) { - String methodName = "createEdgeType"; - String s = id.pname(); - OrientEdgeType e = g.getEdgeType(s); - if ( e == null ) { - doLog(methodName, "Create edge", s); - g.createEdgeType(s); + int max = fields.length - 1; + int current = 0; + StringBuffer buf = new StringBuffer("CREATE TABLE IF NOT EXISTS "); + buf.append(tableName); + buf.append(" ("); + for (String s : fields) { + buf.append(s); + if ( current++ < max) buf.append(", "); } + buf.append(") WITH CLUSTERING ORDER BY (ducc_dbid desc)"); + return buf.toString(); } -// void createVertexType(OrientGraphNoTx g, DbVertex id) -// { -// String methodName = "createVertexType"; -// String s = id.pname(); -// OrientVertexType e = g.getVertexType(s); -// if ( e == null ) { -// doLog(methodName, "Create vertex " + s); -// e = g.createVertexType(s, DuccVertexBase.VBase.pname()); -// } -// } - - void createVertices(OrientGraphNoTx g) - { - String methodName = "createVertices"; - - for (DbVertex v : DbVertex.values()) { - String s = v.pname(); - OrientVertexType vt = g.getVertexType(s); - - - if ( vt == null ) { - if ( v.parent() == null ) { - vt = g.createVertexType(s); - doLog(methodName, "Create vertex " + s); - } else { - vt = g.createVertexType(s, v.parent().pname()); - doLog(methodName, "Create vertex ", s, "subclass from", v.parent().pname()); - } - - Index[] ndx = v.indices(); - if ( ndx != null ) { - for ( Index i : ndx ) { - OProperty p = vt.createProperty(i.propname(), i.type()); - p.setMandatory(true); - doLog(methodName, "Create property", i.propname(), "on", s); - } - } - - // String sql = "create index " + i.pname() + " on " + s + "(" + i.propname() + " by key) notunique"; - // g.command(new OCommandSQL(sql)).execute(); - // doLog(methodName, "(sql)Created index", i.pname(), "on class", s, "for", i.propname()); - // //g.createKeyIndex(i.propname(), Vertex.class, new Parameter("type", "NOTUNIQUE_HASH_INDEX")); - // doLog(methodName, "(java)Created index on class Vertex for", i.propname()); - // } - // } - - } - } - - if ( true ) return; - String sql = "create index D.VBASE_0 on VWork(ducc_dbid, ducc_dbcat) NOTUNIQUE"; - g.command(new OCommandSQL(sql)).execute(); - - String sql1 = "create index D.VBASE_ID on VWork(ducc_dbid) NOTUNIQUE"; - g.command(new OCommandSQL(sql1)).execute(); - - String sql2 = "create index D.VBASE_CAT on VWork(ducc_dbcat) NOTUNIQUE"; - g.command(new OCommandSQL(sql2)).execute(); - - String sql3 = "create index D.NODE on VRmNode(ducc_dbnode) NOTUNIQUE"; - g.command(new OCommandSQL(sql3)).execute(); - - } - - void createEdges(OrientGraphNoTx g) - { - String methodName = "createEdges"; - for (DbEdge e : DbEdge.values()) { - String s = e.pname(); - OrientEdgeType et = g.getEdgeType(s); - if ( et == null ) { - if ( e.parent() == null ) { - et = g.createEdgeType(s); - doLog(methodName, "Create edge " + s); - } else { - et = g.createEdgeType(s, e.parent().pname()); - doLog(methodName, "Create edge ", s, "subclass from", e.parent().pname()); - } - } - } - } - void createSchema() { - OrientGraphNoTx g = factory.getNoTx(); + String methodName = "createSchema"; + + // A 'keyspace' is what we usually think of as a database. + session.execute("CREATE KEYSPACE IF NOT EXISTS ducc WITH replication = {'class':'SimpleStrategy', 'replication_factor':1};"); try { - createVertices(g); - createEdges(g); - } finally { - g.shutdown(); - } - } + List<SimpleStatement>rmSchema = RmStatePersistence.mkSchema(); + for ( SimpleStatement s : rmSchema ) { + doLog(methodName, "EXECUTE STATEMENT:", s.toString()); + session.execute(s); + } - // void createSchemax() - // { - // String methodName = "createSchema"; - // OrientGraphNoTx g = factory.getNoTx(); - - // String base = DuccVertexBase.VBase.pname(); - // OrientVertexType e = g.getVertexType(base); - // if ( e == null ) { - // doLog(methodName, "Create base vertex class " + base); - // e = g.createVertexType(base); - // OProperty p = e.createProperty(DbConstants.DUCCID, OType.LONG); - // p.setMandatory(true); - // OProperty p2 = e.createProperty(DbConstants.DUCC_DBCAT, OType.STRING); - // p2.setMandatory(true); - - // String sql = "create index i_ducc_dbid on " + base + "(" + DbConstants.DUCCID + ") notunique"; - // g.command(new OCommandSQL(sql)).execute(); - // doLog(methodName, "(sql)Created index i_ducc_dbid on class " + base + " for " + DbConstants.DUCCID); - - // sql = "create index i_ducc_dbcat on " + base + "(" + DbConstants.DUCC_DBCAT + ") notunique"; - // g.command(new OCommandSQL(sql)).execute(); - // doLog(methodName, "(sql)Created index i_ducc_dbcat on class " + base + " for " + DbConstants.DUCC_DBCAT); + // + // String[] rmSchema = RmStatePersistence.mkSchemaItems(); + // String cql = DbUtil.mkTableCreate("ducc.rmnodes", rmSchema); + // doLog(methodName, "CQL:", cql); + // session.execute(cql); + + List<SimpleStatement>smSchema = StateServicesDb.mkSchema(); + for ( SimpleStatement s : smSchema ) { + doLog(methodName, "EXECUTE STATEMENT:", s.toString()); + session.execute(s); + } - // } - - // for ( DbVertex o : DbVertex.values() ) { - // createVertexType(g, o); - // } - // for ( DbEdge o : DbEdge.values() ) { - // createEdgeType(g, o); - // } - - // g.shutdown(); - // } - boolean createPlocalDatabase() - throws Exception - { - boolean ret = false; - try { - factory = new OrientGraphFactory(dburl, "admin", "admin"); - createSchema(); - ret = true; +// String[] smSchemaReg = StateServicesDb.mkSchemaForReg(); +// cql = DbUtil.mkTableCreate("ducc.smreg", smSchemaReg); +// doLog(methodName, "CQL:", cql); +// session.execute(cql); +// cql = "CREATE INDEX IF NOT EXISTS ON ducc.smreg(active)"; +// session.execute(cql); +// +// String[] smSchemaMeta = StateServicesDb.mkSchemaForMeta(); +// cql = DbUtil.mkTableCreate("ducc.smmeta", smSchemaMeta); +// doLog(methodName, "CQL:", cql); +// session.execute(cql); +// cql = "CREATE INDEX IF NOT EXISTS ON ducc.smmeta(active)"; +// session.execute(cql); + + List<SimpleStatement>orSchema = HistoryManagerDb.mkSchema(); + for ( SimpleStatement s : orSchema ) { + doLog(methodName, "EXECUTE STATEMENT:", s.toString()); + session.execute(s); + } + } catch ( Exception e ) { - e.printStackTrace(); - } finally { - factory.close(); + doLog(methodName, "Cannot create schema:", e); } - return ret; + } - + /** * Create the database and initialize the schema. This is intended to be called only from Main at * system startup, to insure all users of the db have a db when they start. @@ -245,36 +179,8 @@ public class DbCreate boolean createDatabase() throws Exception { - String methodName = "createDatabase"; - - if ( adminpw == null ) { - adminpw = DbManager.dbPassword(); - } + //String methodName = "createDatabase"; - try { - admin = new OServerAdmin(dburl); - admin.connect(adminid, adminpw); // connect to the server - - if ( ! admin.existsDatabase("plocal") ) { - doLog(methodName, "Database " + dburl + " does not exist, attempting to create it."); - admin.createDatabase("graph", "plocal"); - - if ( ! admin.existsDatabase() ) { - doLog(methodName, "Cannot create database " + dburl); - return false; - } - factory = new OrientGraphFactory(dburl); - if ( factory == null ) { - doLog(methodName, "Cannot create graph factory for " + dburl); - return false; - } - - createSchema(); - } - } finally { - if ( admin != null ) admin.close(); - if ( factory != null ) factory.close(); - } return true; } @@ -284,17 +190,20 @@ public class DbCreate System.out.println("Usage: DbCreate <database url>"); System.exit(1); } + + DbCreate client = null; try { - DbCreate dbc = new DbCreate(args[0]); - if ( ! dbc.createDatabase() ) { - System.out.println("Could not create database or schema for " + args[0]); - System.exit(1); - } - } catch ( Exception e ) { + client = new DbCreate(args[0]); + client.connect(); + client.createSchema(); + } catch ( Throwable e ) { System.out.println("Errors creating database"); e.printStackTrace(); System.exit(1); + } finally { + if ( client != null ) client.close(); } + System.exit(0); }