Author: challngr Date: Thu Dec 10 20:13:26 2015 New Revision: 1719195 URL: http://svn.apache.org/viewvc?rev=1719195&view=rev Log: UIMA-4577 Tweaks so database classes can be used in CLI utilities. Implement rm_qoccupancy using the DB as proof of concept.
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy?rev=1719195&r1=1719194&r2=1719195&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy Thu Dec 10 20:13:26 2015 @@ -21,59 +21,95 @@ import os import sys +import glob import subprocess from ducc_util import DuccUtil class DuccRmQOccupancy(DuccUtil): - def format_shares(self, sh): - for s in sh: - if ( s['blacklisted'] ): - pass - else: - print '%19s J[%8d] S[%8d] O[%d] II[%8d] IR[%8d] E[%5s] P[%5s] F[%5s] I[%5s]' % ('', s['jobid'], s['shareid'], s['order'], s['investment-init'], s['investment-run'], - s['evicted'], s['purged'], s['fixed'], s['initialized']) - - - def format_machines(self, lines): - - print("%20s %11s %6s %6s %15s %10s %5s %6s" % ("Node", "Blacklisted", "Online", "Status", "Nodepool", "Memory", "Order", "Free")) - - for m in lines: - print "%20s %11s %6s %6s %15s %10s %5s %6s" % (m['name'], m['blacklisted'], m['online'], m['status'], m['nodepool'], m['memory'], m['order'], m['shares-free']) - if ( len(m['shares']) != 0 ) : - self.format_shares(m['shares']) - print '' + def format(self, nodes, shares): + print("%20s %11s %6s %6s %15s %10s %6s %6s %6s %8s %7s %10s %8s" % ("Node", "Blacklisted", "Online", "Status", "Nodepool", "Memory", "Order", "Free", "In-Use", "Np-InUse", "Quantum", "Reservable", "Classes")) + print("%20s %11s %6s %6s %15s %10s %6s %6s %6s %8s %7s %10s %8s" % ("----", "-----------", "------", "------", "--------", "------", "-----", "----", "------", "--------", "-------", "----------", "-------")) + for n in nodes: + if (n['responsive']): + status = 'up' + else: + status = 'down' + print "%20s %11s %6s %6s %15s %10s %3s(Q) %6s %6s %8s %7s %10s %8s" % (n['name'], n['blacklisted'], n['online'], status, n['nodepool'], + n['memory'], n['share_order'], n['shares_left'], n['assignments'], + n['np_assignments'], n['quantum'], n['reservable'], n['classes']) + if ( shares.has_key(n['name']) ): + for s in shares[n['name']]: + fmt = '%19s ' + s['jobtype'] +'[%8d] S[%8d] O[%d] II[%8d] IR[%8d] E[%5s] P[%5s] F[%5s] S[%10s]' + state = s['state'] + if ( state == 'null' ): + state = "Assigned" + print fmt % ('', s['job_id'], s['ducc_dbid'], s['share_order'], s['init_time'], s['investment'], s['evicted'], s['purged'], s['fixed'], state) - def format(self, lines): - qoccupancy = eval(lines) - self.format_machines(qoccupancy) + print '' + # Given DUCC_HOME, a directory, and part of the name of a jar, find the actual name of the jar which will + # likeley be versioned + def resolve_jar(self, DH, dirname, basename): + partial = DH + '/' + dirname + '/'+ basename + '*' + ret = glob.glob(partial) + return ret[0] + def main(self, argv): if ( argv == '-h' or argv == '-?' or argv == '--help' ): print 'rm_qload queries and formats the current state of the RM scheduling tables. It takes no parameters.' sys.exit(1); - DUCC_JVM_OPTS = ' -Dducc.deploy.configuration=' + self.DUCC_HOME + "/resources/ducc.properties " + DH = self.DUCC_HOME + CP = [self.resolve_jar(DH, '/lib/uima-ducc', 'uima-ducc-database'), + self.resolve_jar(DH, '/lib/uima-ducc', 'uima-ducc-common'), + DH + '/lib/cassandra/*', + DH + '/lib/apache-log4j/*', + DH + '/lib/guava/*', + self.resolve_jar(DH, '/cassandra-server/lib', 'slf4j-api'), + self.resolve_jar(DH, '/apache-uima/apache-activemq/lib', 'slf4j-log4j12'), + ] + os.environ['CLASSPATH'] = ':'.join(CP) + + DUCC_JVM_OPTS = '' DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -DDUCC_HOME=' + self.DUCC_HOME - DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -Dducc.head=' + self.ducc_properties.get('ducc.head') + DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -Dducc.rm.persistence.impl=org.apache.uima.ducc.database.RmStatePersistence' - CMD = [self.java(), DUCC_JVM_OPTS, 'org.apache.uima.ducc.common.main.DuccRmAdmin', '--qoccupancy'] + dbn = self.ducc_properties.get('ducc.database.host') + CMD = [self.java(), DUCC_JVM_OPTS, 'org.apache.uima.ducc.database.RmShareState', dbn] + CMD = ' '.join(CMD) + lines = '' + proc = subprocess.Popen(CMD, bufsize=0, stdout=subprocess.PIPE, shell=True) + for line in proc.stdout: + lines = lines + line + + shares = eval(lines) + + shares_by_machine = {} + for share in shares: + k = share['node'] + if ( shares_by_machine.has_key(k) ): + share_list = shares_by_machine[k] + else: + share_list = [] + shares_by_machine[k] = share_list + share_list.append(share) + + CMD = [self.java(), DUCC_JVM_OPTS, 'org.apache.uima.ducc.database.RmNodeState', dbn] CMD = ' '.join(CMD) lines = '' proc = subprocess.Popen(CMD, bufsize=0, stdout=subprocess.PIPE, shell=True) for line in proc.stdout: lines = lines + line - if 'not yet initialized' in lines: - print lines - return - - self.format(lines) + nodes = eval(lines) + nodes = sorted(nodes, key=lambda n: n["name"]) + + self.format(nodes, shares_by_machine) return Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java?rev=1719195&r1=1719194&r2=1719195&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java Thu Dec 10 20:13:26 2015 @@ -19,6 +19,7 @@ package org.apache.uima.ducc.common.persistence.rm; +import java.util.List; import java.util.Map; import java.util.Properties; @@ -167,7 +168,7 @@ public interface IRmPersistence /** * Fetch all machines in the database. * - * @return A map of properties objects containing full details about the machines, + * @return A map of map objects containing full details about the machines, * keyed on machine name. If there are no machines found in the db, * an empty map is returned. * @@ -177,6 +178,31 @@ public interface IRmPersistence public Map<String, Map<String, Object>> getAllMachines() throws Exception; /** + * Fetch all shares in the database. + * + * @return A map of map objects containing full details about the shares. + * The key is node:jobid:shareid concatenated as a string, as it takes all of these + * to uniquely identify a share. If there are no shares found in the db, + * an empty map is returned. + * + * @throws Exception. Anything that goes wrong throws. Usually the + * throw will originate in the DB because of some DB issue. + */ + public Map<String, Map<String, Object>> getAllShares() throws Exception; + + /** + * Fetch all the load records. These are relatively short, one per job/reservation/etc + * in the database, summarizing the resources they have vs the resources they want in + * a perfect world. + * + * @return A list of map objects containing load information. + * + * @throws Exception. Anything that goes wrong throws. Usually the + * throw will originate in the DB because of some DB issue. + */ + public List<Map<String, Object>> getLoad() throws Exception; + + /** * A new job arrives (or is recovered after restart). */ public void addJob(IDbJob j) throws Exception; Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java?rev=1719195&r1=1719194&r2=1719195&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/NullRmStatePersistence.java Thu Dec 10 20:13:26 2015 @@ -18,7 +18,9 @@ */ package org.apache.uima.ducc.common.persistence.rm; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Properties; @@ -48,6 +50,8 @@ public class NullRmStatePersistence impl public void updateShare(String node, DuccId shareid, DuccId jobid, long investment, String state, long init_time, long pid) {} public Properties getMachine(String id) { return null; } public Map<String, Map<String, Object>> getAllMachines() { return new HashMap<String, Map<String, Object>>(); } + public Map<String, Map<String, Object>> getAllShares() { return new HashMap<String, Map<String, Object>>(); } + public List<Map<String, Object>> getLoad() { return new ArrayList<Map<String, Object>>(); } public void addJob(IDbJob j ) {} public void deleteJob(IDbJob j ) {} public void updateDemand(IDbJob j) {} Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java?rev=1719195&r1=1719194&r2=1719195&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java Thu Dec 10 20:13:26 2015 @@ -50,7 +50,7 @@ public class DbHandle String methodName = "execute"; long now = System.currentTimeMillis(); ResultSet ret = manager.execute(sql); - logger.info(methodName, null, "Time to execute", System.currentTimeMillis() - now); + if ( manager.noisy ) logger.info(methodName, null, "Time to execute", System.currentTimeMillis() - now); return ret; } @@ -76,7 +76,7 @@ public class DbHandle BoundStatement bound = boundStatement.bind(fields); return execute(bound); } finally { - logger.info(methodName, null, "Time to execute prepared statement:", ps.getQueryString(), System.currentTimeMillis() - now); + if ( manager.noisy ) logger.info(methodName, null, "Time to execute prepared statement:", ps.getQueryString(), System.currentTimeMillis() - now); } } @@ -253,7 +253,6 @@ public class DbHandle PreparedStatement prepare(String cql) { String methodName = "prepare"; - logger.info(methodName, null, "Preparing", cql); return manager.prepare(cql); } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java?rev=1719195&r1=1719194&r2=1719195&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java Thu Dec 10 20:13:26 2015 @@ -44,9 +44,11 @@ import com.datastax.driver.core.policies public class DbManager { static final String URL_PROPERTY = "ducc.database.url"; + static final String NOISE_PROPERTY = "ducc.database.noisy"; private static String db_id = null; private static String db_pw = null; + boolean noisy = true; String dburl; DuccLogger logger; @@ -59,6 +61,8 @@ public class DbManager { this.dburl = dburl; this.logger = logger; + + if ( System.getProperty(NOISE_PROPERTY) != null ) noisy = false; } boolean checkForDatabase() @@ -97,7 +101,6 @@ public class DbManager // will throw sometims, so we can assume // we're allowed to continue if control passes down. - logger.info(methodName, null, "Attach to", dburl, "as", db_id, db_pw); PlainTextAuthProvider auth = new PlainTextAuthProvider(db_id, db_pw); // throws if no good ReconnectionPolicy rp = new ConstantReconnectionPolicy(10000); // if we lose connection, keep trying every 10 seconds @@ -108,17 +111,18 @@ public class DbManager .build(); Metadata metadata = cluster.getMetadata(); - logger.info(methodName, null, "Connected to cluster:", metadata.getClusterName()); - - for ( Host host : metadata.getAllHosts() ) { - logger.info(methodName, null, "Datatacenter:", host.getDatacenter(), "Host:", host.getAddress(), "Rack:", host.getRack()); - } + if ( noisy) { + logger.info(methodName, null, "Connected to cluster:", metadata.getClusterName()); + for ( Host host : metadata.getAllHosts() ) { + logger.info(methodName, null, "Datatacenter:", host.getDatacenter(), "Host:", host.getAddress(), "Rack:", host.getRack()); + } + } } public synchronized void shutdown() { String methodName = "closeDatabase"; - logger.info(methodName, null, "Closing the database."); + if ( noisy ) logger.info(methodName, null, "Closing the database."); if ( cluster != null ) cluster.close(); cluster = null; session = null; @@ -148,9 +152,9 @@ public class DbManager { String methodName = "execute"; if ( logger.isDebug() ) { - logger.info(methodName, null, "EXECUTE CQL:", cql); + if ( noisy ) logger.info(methodName, null, "EXECUTE CQL:", cql); } else { - logger.info(methodName, null, "EXECUTE CQL:", truncateText(cql)); + if ( noisy ) logger.info(methodName, null, "EXECUTE CQL:", truncateText(cql)); } return session.execute(cql); } @@ -163,7 +167,7 @@ public class DbManager ResultSet execute(SimpleStatement s) { String methodName = "execute"; - logger.info(methodName, null, "EXECUTE STATEMENT:", truncateText(s.getQueryString())); + if ( noisy ) logger.info(methodName, null, "EXECUTE STATEMENT:", truncateText(s.getQueryString())); return session.execute(s); } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java?rev=1719195&r1=1719194&r2=1719195&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmNodeState.java Thu Dec 10 20:13:26 2015 @@ -21,11 +21,14 @@ package org.apache.uima.ducc.database; import java.util.Map; +import org.apache.uima.ducc.common.persistence.IDbProperty; import org.apache.uima.ducc.common.persistence.rm.IRmPersistence; +import org.apache.uima.ducc.common.persistence.rm.IRmPersistence.RmNodes; import org.apache.uima.ducc.common.persistence.rm.NullRmStatePersistence; import org.apache.uima.ducc.common.persistence.rm.RmPersistenceFactory; import org.apache.uima.ducc.common.utils.DuccLogger; + public class RmNodeState { DuccLogger logger = DuccLogger.getLogger(RmNodeState.class, "State"); @@ -35,33 +38,20 @@ public class RmNodeState { this.dburl = dburl; } - + void run() throws Exception { + System.setProperty(DbManager.NOISE_PROPERTY, "false"); IRmPersistence persistence = RmPersistenceFactory.getInstance(this.getClass().getName(), "RM"); if ( persistence instanceof NullRmStatePersistence ) { System.out.println("Cannot get viable RM Persistance isntance."); return; } - + try { - for ( int i = 0; i < 10; i++ ) { - Map<String, Map<String, Object>> state = persistence.getAllMachines(); - for ( String node : state.keySet() ) { - StringBuffer buf = new StringBuffer(node); - buf.append(": "); - Map<String, Object> st = state.get(node); - for ( String k : st.keySet() ) { - buf.append(k); - buf.append("["); - buf.append(st.get(k).toString()); - buf.append("] "); - } - System.out.println(buf.toString()); - } - Thread.sleep(2000); - } + Map<String, Map<String, Object>> state = persistence.getAllMachines(); + System.out.println(toJson(state)); } catch ( Exception e ) { e.printStackTrace(); } finally { @@ -70,6 +60,45 @@ public class RmNodeState } } + String toJson(Map<String, Map<String, Object>> nodes) + { + StringBuffer buf = new StringBuffer("["); + for ( Map<String, Object> vals : nodes.values() ) { + buf.append("{"); + for ( IDbProperty p : RmNodes.values() ) { + if ( p.isMeta() ) continue; + if ( p.isPrivate() ) continue; + buf.append("'"); + buf.append(p.pname()); + buf.append("'"); + buf.append(":"); + switch(p.type()) { + case String: + buf.append("'"); // must quote strings + buf.append(vals.get(p.columnName())); + buf.append("'"); + break; + case Boolean: + boolean bv = (boolean) vals.get(p.columnName()); + buf.append(bv ? "True" : "False"); // must pythonify the booleans + break; + case Integer: + case Long: + case Double +: buf.append(vals.get(p.columnName()).toString()); + break; + default: + // RmNodes doesn't use other types + break; + } + buf.append(","); + } + buf.append("},\n"); + } + buf.append("]"); + return buf.toString(); + } + public static void main(String[] args) { if ( args.length != 1 ) { Added: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java?rev=1719195&view=auto ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java (added) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmQLoad.java Thu Dec 10 20:13:26 2015 @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. +*/ + +package org.apache.uima.ducc.database; + +import java.util.List; +import java.util.Map; + +import org.apache.uima.ducc.common.persistence.IDbProperty; +import org.apache.uima.ducc.common.persistence.rm.IRmPersistence; +import org.apache.uima.ducc.common.persistence.rm.IRmPersistence.RmLoad; +import org.apache.uima.ducc.common.persistence.rm.NullRmStatePersistence; +import org.apache.uima.ducc.common.persistence.rm.RmPersistenceFactory; +import org.apache.uima.ducc.common.utils.DuccLogger; + + +public class RmQLoad +{ + DuccLogger logger = DuccLogger.getLogger(RmQLoad.class, "State"); + String dburl = null; + + RmQLoad(String dburl) + { + this.dburl = dburl; + } + + void run() + throws Exception + { + System.setProperty(DbManager.NOISE_PROPERTY, "false"); + IRmPersistence persistence = RmPersistenceFactory.getInstance(this.getClass().getName(), "RM"); + if ( persistence instanceof NullRmStatePersistence ) { + System.out.println("Cannot get viable RM Persistance isntance."); + return; + } + + try { + List<Map<String, Object>> state = persistence.getLoad(); + System.out.println(toJson(state)); + } catch ( Exception e ) { + e.printStackTrace(); + } finally { + // In "real life" you don't need to, and shouldn't, close the persistence until the process is ready to exit. + persistence.close(); + } + } + + String toJson(List<Map<String, Object>> nodes) + { + StringBuffer buf = new StringBuffer("["); + for ( Map<String, Object> vals : nodes ) { + buf.append("{"); + for ( IDbProperty p : RmLoad.values() ) { + if ( p.isMeta() ) continue; + if ( p.isPrivate() ) continue; + buf.append("'"); + buf.append(p.pname()); + buf.append("'"); + buf.append(":"); + switch(p.type()) { + case String: + case UUID: + buf.append("'"); // must quote strings + buf.append(vals.get(p.columnName())); + buf.append("'"); + break; + case Boolean: + boolean bv = (boolean) vals.get(p.columnName()); + buf.append(bv ? "True" : "False"); // must pythonify the booleans + break; + case Integer: + case Long: + case Double: + buf.append(vals.get(p.columnName()).toString()); + break; + default: + // RmNodes doesn't use other types + break; + } + buf.append(","); + } + buf.append("},\n"); + } + buf.append("]"); + return buf.toString(); + } + + public static void main(String[] args) + { + if ( args.length != 1 ) { + System.out.println("Usage: RmQLoad <dburl>"); + System.exit(1); + } + System.setProperty(DbManager.URL_PROPERTY, args[0]); + + RmQLoad rns = new RmQLoad(args[0]); + try { + rns.run(); + } catch ( Exception e ) { + e.printStackTrace(); + } + } +} Added: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java?rev=1719195&view=auto ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java (added) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmShareState.java Thu Dec 10 20:13:26 2015 @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. +*/ + +package org.apache.uima.ducc.database; + +import java.util.Map; + +import org.apache.uima.ducc.common.persistence.IDbProperty; +import org.apache.uima.ducc.common.persistence.rm.IRmPersistence; +import org.apache.uima.ducc.common.persistence.rm.IRmPersistence.RmShares; +import org.apache.uima.ducc.common.persistence.rm.NullRmStatePersistence; +import org.apache.uima.ducc.common.persistence.rm.RmPersistenceFactory; +import org.apache.uima.ducc.common.utils.DuccLogger; + + +public class RmShareState +{ + DuccLogger logger = DuccLogger.getLogger(RmShareState.class, "State"); + String dburl = null; + + RmShareState(String dburl) + { + this.dburl = dburl; + } + + void run() + throws Exception + { + System.setProperty(DbManager.NOISE_PROPERTY, "false"); + IRmPersistence persistence = RmPersistenceFactory.getInstance(this.getClass().getName(), "RM"); + if ( persistence instanceof NullRmStatePersistence ) { + System.out.println("Cannot get viable RM Persistance isntance."); + return; + } + + try { + Map<String, Map<String, Object>> state = persistence.getAllShares(); + System.out.println(toJson(state)); + } catch ( Exception e ) { + e.printStackTrace(); + } finally { + // In "real life" you don't need to, and shouldn't, close the persistence until the process is ready to exit. + persistence.close(); + } + } + + String toJson(Map<String, Map<String, Object>> nodes) + { + StringBuffer buf = new StringBuffer("["); + for ( Map<String, Object> vals : nodes.values() ) { + buf.append("{"); + for ( IDbProperty p : RmShares.values() ) { + if ( p.isMeta() ) continue; + if ( p.isPrivate() ) continue; + buf.append("'"); + buf.append(p.pname()); + buf.append("'"); + buf.append(":"); + switch(p.type()) { + case String: + case UUID: + buf.append("'"); // must quote strings + buf.append(vals.get(p.columnName())); + buf.append("'"); + break; + case Boolean: + boolean bv = (boolean) vals.get(p.columnName()); + buf.append(bv ? "True" : "False"); // must pythonify the booleans + break; + case Integer: + case Long: + case Double: + buf.append(vals.get(p.columnName()).toString()); + break; + default: + // RmNodes doesn't use other types + break; + } + buf.append(","); + } + buf.append("},\n"); + } + buf.append("]"); + return buf.toString(); + } + + public static void main(String[] args) + { + if ( args.length != 1 ) { + System.out.println("Usage: RmShareState <dburl>"); + System.exit(1); + } + System.setProperty(DbManager.URL_PROPERTY, args[0]); + + RmShareState rns = new RmShareState(args[0]); + try { + rns.run(); + } catch ( Exception e ) { + e.printStackTrace(); + } + } +} Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java?rev=1719195&r1=1719194&r2=1719195&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java Thu Dec 10 20:13:26 2015 @@ -203,7 +203,7 @@ public class RmStatePersistence } catch ( Exception e ) { logger.error(methodName, null, "Problem setting properties", e); } finally { - logger.info(methodName, null, "Total time to update properties on", System.currentTimeMillis() - now); + logger.debug(methodName, null, "Total time to update properties on", System.currentTimeMillis() - now); } @@ -285,7 +285,39 @@ public class RmStatePersistence ret.put((String)mach.get(RmNodes.Name.pname()), mach); } return ret; - } + } + + public Map<String, Map<String, Object>> getAllShares() + throws Exception + { + //String methodName = "getAllShares"; + Map<String, Map<String, Object>> ret = new HashMap<String, Map<String, Object>>(); + String cql = "SELECT * FROM " + RM_SHARE_TABLE; + DbHandle h = dbManager.open(); + ResultSet rs = h.execute(cql); + for ( Row r : rs ) { + + Map<String, Object> share = DbUtil.getProperties(RmShares.values(), r); + String key = share.get(RmShares.Node.pname()) + ":" + share.get(RmShares.DuccDbid.pname()) + ":" + share.get(RmShares.JobId.pname()); + ret.put(key, share); + } + return ret; + } + + public List<Map<String, Object>> getLoad() + throws Exception + { + //String methodName = "getLoad"; + List<Map<String, Object>> ret = new ArrayList<Map<String, Object>>(); + String cql = "SELECT * FROM " + RM_LOAD_TABLE; + DbHandle h = dbManager.open(); + ResultSet rs = h.execute(cql); + for ( Row r : rs ) { + Map<String, Object> job = DbUtil.getProperties(RmLoad.values(), r); + ret.add(job); + } + return ret; + } public void addJob(IDbJob j) throws Exception