Author: challngr
Date: Wed Nov 11 14:45:37 2015
New Revision: 1713860

URL: http://svn.apache.org/viewvc?rev=1713860&view=rev
Log:
UIME-4577 DB support updates.

Modified:
    uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
    uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
    uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml
    
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
    
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java
    
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
    
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
    
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
    
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
    
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Wed Nov 11 14:45:37 
2015
@@ -55,7 +55,7 @@ class CheckDucc(DuccUtil):
         if ( self.db_bypass == True ):
             return True
 
-        ret = self.db_alive(3)
+        ret = self.db_alive(1)
         if ( ret ):
             print 'The database is running'
         else:

Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Wed Nov 11 
14:45:37 2015
@@ -208,6 +208,11 @@ class DuccUtil(DuccBase):
         if ( dbnode == None ):
             print 'No database location defined.'
             return False
+
+        pidfile = self.DUCC_HOME + '/state/cassandra.pid'
+        if ( not os.path.exists(pidfile) ):
+            print 'Database pid file does not exist.  Checking DB 
connectivity.'
+
         # get our log4j config into the path to shut up noisy logging
         os.environ['CLASSPATH'] = os.environ['CLASSPATH'] + ':' + 
self.DUCC_HOME + '/resources'
         
@@ -269,11 +274,11 @@ class DuccUtil(DuccBase):
             return True
 
         pidfile = self.DUCC_HOME + '/state/cassandra.pid'
-
-        # for cassandra, just send it a terminate signal.  a pidfile is 
written on startup
-        CMD = ['kill', '-TERM', '`cat ' + pidfile + '`']
-        CMD = ' '.join(CMD)
-        os.system(CMD)
+        if ( os.path.exists(pidfile) ):
+            # for cassandra, just send it a terminate signal.  a pidfile is 
written on startup
+            CMD = ['kill', '-TERM', '`cat ' + pidfile + '`']
+            CMD = ' '.join(CMD)
+            os.system(CMD)
 
     def find_netstat(self):
         # don't you wish people would get together on where stuff lives?

Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml Wed Nov 11 
14:45:37 2015
@@ -140,12 +140,13 @@
     </layout> 
   </appender> 
   
-   <category name="org.apache.uima.ducc.rm.scheduler" additivity="true">
+   <category name="org.apache.uima.ducc.rm" additivity="true">
      <priority value="debug"/>
      <appender-ref ref="rmlog" /> 
    </category>
 
-   <category name="org.apache.uima.ducc.rm.JobManagerConverter" 
additivity="true">
+   <!-- Please keep these for reference as they focus on specific components 
of interest during debug. -->
+   <!-- category name="org.apache.uima.ducc.rm.JobManagerConverter" 
additivity="true">
      <priority value="info"/>
      <appender-ref ref="rmlog" /> 
    </category>
@@ -158,7 +159,7 @@
    <category name="org.apache.uima.ducc.rm.ResourceManagerComponent" 
additivity="true">
      <priority value="info"/>
      <appender-ref ref="rmlog" /> 
-   </category>
+   </category -->
 
    <category name="org.apache.uima.ducc.orchestrator" additivity="true">
      <priority value="info"/>
@@ -240,6 +241,11 @@
      <appender-ref ref="jp-log" /> 
    </category>
 
+   <!-- cassandra client -->
+   <category name="com.datastax.driver">
+     <priority value="warn"/>
+   </category>
+
    <category name="org.apache.uima.ducc.database.DbLoader" additivity="true">
      <priority value="info"/>
      <appender-ref ref="db-loader-log" /> 

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
 Wed Nov 11 14:45:37 2015
@@ -33,6 +33,8 @@ import com.datastax.driver.core.Prepared
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Session;
 import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.policies.ConstantReconnectionPolicy;
+import com.datastax.driver.core.policies.ReconnectionPolicy;
 
 /**
  * Provide a common point for contacting the db, acquiring sessions/handles to 
it, managing the db,
@@ -90,9 +92,11 @@ public class DbManager
         String pw = dbPassword();
         PlainTextAuthProvider auth = new PlainTextAuthProvider("ducc", pw);
 
+        ReconnectionPolicy rp = new ConstantReconnectionPolicy(10000);  // if 
we lose connection, keep trying every 10 seconds
         cluster = Cluster.builder()
             .withAuthProvider(auth)
             .addContactPoint(dburl)
+            .withReconnectionPolicy(rp)
             .build();
 
         Metadata metadata = cluster.getMetadata();
@@ -107,7 +111,7 @@ public class DbManager
     {
        String methodName = "closeDatabase";
         logger.info(methodName, null, "Closing the database.");
-        cluster.close();        
+        if ( cluster != null ) cluster.close();        
         cluster = null;
         session = null;
     }

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java
 Wed Nov 11 14:45:37 2015
@@ -20,12 +20,15 @@
 package org.apache.uima.ducc.database;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import org.apache.uima.ducc.common.persistence.IDbProperty;
 import org.apache.uima.ducc.common.persistence.IDbProperty.Type;
 
+import com.datastax.driver.core.Row;
+
 /**
  * Static common helper methods.
  *
@@ -227,6 +230,51 @@ class DbUtil
     }
 
     /**
+     * Common code to pull things from a row according to the schema, into a 
map
+     */
+    static Map<String, Object> getProperties(IDbProperty[] props, Row r)
+    {
+        Map<String, Object> ret = new HashMap<String, Object>();
+        for ( IDbProperty p : props ) {
+            if ( p.isPrivate() ) continue;
+            if ( p.isMeta()    ) continue;
+            Object val = null;
+            switch ( p.type() ) {
+
+                case String: 
+                    val = r.getString(p.columnName());
+                    break;
+
+                case Integer: 
+                    val = r.getInt(p.columnName());
+                    break;
+
+                case Long: 
+                    val = r.getLong(p.columnName());
+                    break;
+
+                case Double: 
+                    val = r.getDouble(p.columnName());
+                    break;
+
+                case UUID: 
+                    val = r.getUUID(p.columnName());
+                    break;
+
+                case Boolean: 
+                    val = r.getBool(p.columnName());
+                    break;
+
+                case Blob: 
+                    val = r.getBytes(p.columnName());
+                    break;
+            }
+            if ( val != null ) ret.put(p.pname(), val);
+        }
+        return ret;
+    }
+
+    /**
      * Convert our generic "type" to the right name for this db implementation.
      * We could make Type a magic enum but I want to hide DB specifics, in 
particular,
      * how this database names various java types.

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
 Wed Nov 11 14:45:37 2015
@@ -26,6 +26,7 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.uima.ducc.common.Pair;
 import org.apache.uima.ducc.common.utils.DuccLogger;
@@ -43,6 +44,7 @@ import com.datastax.driver.core.Prepared
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Row;
 import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.exceptions.NoHostAvailableException;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 
@@ -75,24 +77,30 @@ public class HistoryManagerDb
                String methodName = "init";
         boolean ret = true;
         logger.info(methodName, null, "Initializing OR persistence over the 
database");
-        try {
-            if ( dbm != null ) {
-                this.dbManager = dbm;
-            } else {
-                dbManager = new DbManager(dburl, logger);
-                dbManager.init();
-            }
-
-            // prepare some statements
-            DbHandle h = dbManager.open();
-            jobPrepare         = h.prepare("INSERT INTO " + JOB_TABLE + " 
(ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");          
  
-            reservationPrepare = h.prepare("INSERT INTO " + RES_TABLE + " 
(ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");          
  
-            servicePrepare     = h.prepare("INSERT INTO " + SVC_TABLE + " 
(ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");          
  
-            ckptPrepare = h.prepare("INSERT INTO " + CKPT_TABLE + " (id, work, 
p2jmap) VALUES (?, ?, ?);");            
-
-        } catch ( Exception e ) {
-            logger.error(methodName, null, "Cannot open the history 
database:", e);
-            throw e;
+        while ( true ) {
+            try {
+                if ( dbm != null ) {
+                    this.dbManager = dbm;
+                } else {
+                    dbManager = new DbManager(dburl, logger);
+                    dbManager.init();
+                }
+                
+                // prepare some statements
+                DbHandle h = dbManager.open();
+                jobPrepare         = h.prepare("INSERT INTO " + JOB_TABLE + " 
(ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");          
  
+                reservationPrepare = h.prepare("INSERT INTO " + RES_TABLE + " 
(ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");          
  
+                servicePrepare     = h.prepare("INSERT INTO " + SVC_TABLE + " 
(ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");          
  
+                ckptPrepare        = h.prepare("INSERT INTO " + CKPT_TABLE + " 
(id, work, p2jmap) VALUES (?, ?, ?);");            
+                break;
+            } catch ( NoHostAvailableException e ) {
+                logger.error(methodName, null, "Cannot contact database.  
Retrying in 5 seconds.");
+                Thread.sleep(5000);
+            } catch ( Exception e ) {
+                logger.error(methodName, null, "Errors contacting database.  
No connetion made.", e);
+                ret = false;
+                break;
+            }            
         }
         return ret;
        }
@@ -182,7 +190,7 @@ public class HistoryManagerDb
         ByteBuffer buf = ByteBuffer.wrap(bytes);
 
         DbHandle h = dbManager.open();
-        h.saveObject(jobPrepare,  w.getDuccId().getFriendly(), type, 
isHistory, buf);
+        h.saveObject(s,  w.getDuccId().getFriendly(), type, isHistory, buf);
 
         logger.info(methodName, w.getDuccId(), "----------> Time to save", 
type, ":", System.currentTimeMillis() - nowP, "Size:", bytes.length, "bytes."); 
       
     }
@@ -383,6 +391,14 @@ public class HistoryManagerDb
             byte[] bytes = baos.toByteArray();
             ByteBuffer workbuf = ByteBuffer.wrap(bytes);
 
+            if ( logger.isTrace() ) {
+                ConcurrentHashMap<DuccId, IDuccWork> map = work.getMap();
+                for ( DuccId id : map.keySet() ) {
+                    IDuccWork w = map.get(id);
+                    logger.trace(methodName, id, "Checkpointing", 
w.getClass());
+                }
+            }
+
             baos = new ByteArrayOutputStream();
             out= new ObjectOutputStream(baos);
             out.writeObject(processToJob);
@@ -442,6 +458,13 @@ public class HistoryManagerDb
                     DuccWorkReservation res = (DuccWorkReservation) 
work.findDuccWork(DuccType.Reservation, ""+id.getFriendly());
                     if ( r != null ) res.initLogger();
                 }
+
+                // only gets called once per boot and might be useful, let's 
leave at info
+                ConcurrentHashMap<DuccId, IDuccWork> map = work.getMap();
+                for ( DuccId id : map.keySet() ) {
+                    IDuccWork w = map.get(id);
+                    logger.info(methodName, id, "Restored", w.getClass());
+                }
                 
                 ret = new Pair(work, processToJob);
             }

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
 Wed Nov 11 14:45:37 2015
@@ -32,6 +32,7 @@ import org.apache.uima.ducc.common.utils
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Row;
 import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.exceptions.NoHostAvailableException;
 
 /**
  * Manage saving and fetching of transient RM state.  The primary consumer is
@@ -52,13 +53,22 @@ public class RmStatePersistence
     private boolean init(String dburl)
         throws Exception
     {
+       String methodName = "init";
         boolean ret = false;
-        try {
-            dbManager = new DbManager(dburl, logger);
-            dbManager.init();
-            ret = true;
-        } catch ( Exception e ) {
-            throw e;
+        while ( true ) {
+            try {
+                dbManager = new DbManager(dburl, logger);
+                dbManager.init();
+                ret = true;
+                break;
+            } catch ( NoHostAvailableException e ) {
+                logger.error(methodName, null, "Cannot contact database.  
Retrying in 5 seconds.");
+                Thread.sleep(5000);
+            } catch ( Exception e ) {
+                logger.error(methodName, null, "Errors contacting database.  
No connetion made.", e);
+                ret = false;
+                break;
+            }
         }
         return ret;
     }
@@ -180,39 +190,8 @@ public class RmStatePersistence
         DbHandle h = dbManager.open();
         ResultSet rs = h.execute(cql);
         for ( Row r : rs ) {
-            Map<String, Object> mach = new HashMap<String, Object>();
-            // We don't expect any nulls in this table
-            for ( RmNodes n : RmNodes.values() ) {
-                if ( n.isPrivate() ) continue;
-                if ( n.isMeta()    ) continue;
-                switch ( n.type() ) {
-
-                    case String: {
-                        String  v = r.getString(n.columnName());
-                        mach.put(n.pname(), v);
-                        if ( n == RmNodes.Name ) {
-                            ret.put(v, mach);                            
-                        }
-                    }
-                    break;
-
-                    case Integer: {
-                        int v = r.getInt(n.columnName());
-                        mach.put(n.pname(), v);
-                    }
-                    break;
-
-                    case Boolean: {
-                        boolean v = r.getBool(n.columnName());
-                        mach.put(n.pname(), v);
-                    }
-                    break;
-
-                    default:
-                        logger.warn(methodName, null, "Unexpected value in 
db:", n.pname(), "type", n.type(), "is not recognized.");
-                        break;
-                }
-            }
+            Map<String, Object> mach = DbUtil.getProperties(RmNodes.values(), 
r);
+            ret.put((String)mach.get(RmNodes.Name.pname()), mach);
         }
         return ret;
    }

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
 Wed Nov 11 14:45:37 2015
@@ -36,6 +36,7 @@ import org.apache.uima.ducc.common.utils
 import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Row;
 import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.exceptions.NoHostAvailableException;
 
 public class StateServicesDb
     implements IStateServices
@@ -72,13 +73,19 @@ public class StateServicesDb
         if ( dbm != null ) {
             this.dbManager = dbm;
         } else {
-            try {
-                dbManager = new DbManager(dburl, logger);
-                dbManager.init();
-                ret = true;
-            } catch ( Exception e ) {
-                logger.error(methodName, null, "Cannot open the state 
database:", e);
-                throw e;
+            while ( true ) {
+                try {
+                    dbManager = new DbManager(dburl, logger);
+                    dbManager.init();
+                    break;
+                } catch ( NoHostAvailableException e ) {
+                    logger.error(methodName, null, "Cannot contact the 
database host, Retryng connection in 5 seconds.");
+                    Thread.sleep(5000);
+                } catch ( Exception e ) {
+                    logger.error(methodName, null, "Cannot open the 
database.", e);
+                    ret = false;
+                    break;
+                }
             }
         }
 
@@ -112,32 +119,24 @@ public class StateServicesDb
         s.setFetchSize(100);
         DbHandle h = dbManager.open();
         ResultSet rs = h.execute(s);
-        for ( Row r: rs ) {
-            long id = Long.parseLong(r.getString("numeric_id"));
-            DuccId did = new DuccId(id);
-            logger.debug(methodName, did, "Found properties in table", 
tableid);
+        for ( Row r : rs ) {
+            Map<String, Object> rowvals = DbUtil.getProperties(props, r);
             DuccProperties dp = new DuccProperties();
-            for ( IDbProperty p : props ) {
-                String val = null;
-                if ( logger.isTrace() ) {
-                    logger.info(methodName, null, "Fetching", p.pname(), 
"from", tableid);
-                }
-                if ( !(p.isPrivate() || p.isMeta()) ) {    // collect 
non-private / non-meta properties into dp
-                    val = r.getString(p.columnName());                    
-                    if ( val != null ) {
-                        dp.put(p.pname(), val);
-                    }
-                }
-                if ( p.isPrimaryKey() ) {                  // once we find the 
key we set the dp int return val
-                       String k = null;
-                    if ( val == null ) {
-                        k = r.getString(p.columnName());
-                    } else {
-                        k = val;
-                    }
-                    ret.put(Long.parseLong(k), dp);
+            for (String k : rowvals.keySet()) {
+                dp.put(k, rowvals.get(k));
+            }
+
+            // if these don't work the DB is busted.  This set of properties 
is now useless.
+            try {
+                String id = 
r.getString(IStateServices.SvcRegProps.numeric_id.pname());
+                if ( id == null ) {
+                    throw new IllegalStateException("Missing numeric id for 
service properties.");
                 }
+                ret.put(Long.parseLong(id), dp);
+            } catch ( Exception e ) {
+                logger.error(methodName, null, "Fatal error recovering 
properties.  Discarding row.");
             }
+
         }
        return ret;
     }
@@ -354,7 +353,9 @@ public class StateServicesDb
             Object[] updates = new Object[props.size()*2];
             int i = 0;
             for ( IDbProperty k : map.keySet() ) {
-               logger.info(methodName, null, "Updating", k.columnName(), 
"with", map.get(k));
+                if ( logger.isTrace() ) {
+                    logger.trace(methodName, null, "Updating", k.columnName(), 
"with", map.get(k));
+                }
                 updates[i++] = k;
                 updates[i++] = map.get(k);
             }
@@ -409,44 +410,4 @@ public class StateServicesDb
         return ret;
     }
 
-    // static String[] mkSchemaForReg()
-    // {
-
-    //     int size = SvcRegProps.values().length;
-    //     String[] ret = new String[size];
-
-    //     int ndx = 0;
-
-    //     for ( SvcRegProps p: SvcRegProps.values() ) {
-    //         String s = p.pname();
-    //         s = s + " " + DbUtil.typeToString(p.type());
-    //         if ( p.isPrimaryKey() ) {
-    //             s = s + " PRIMARY KEY";
-    //         }
-    //         ret[ndx++] = s;
-    //     }
-    //     return ret;
-
-    // }
-
-    // static String[] mkSchemaForMeta()
-    // {
-    //     int size = SvcMetaProps.values().length;
-    //     String[] ret = new String[size];
-
-    //     int ndx = 0;
-
-    //     for ( SvcMetaProps p: SvcMetaProps.values() ) {
-    //         String s = p.pname();
-    //         s = s + " " + DbUtil.typeToString(p.type());
-    //         if ( p.isPrimaryKey() ) {
-    //             s = s + " PRIMARY KEY";
-    //         }
-    //         ret[ndx++] = s;
-    //     }
-
-    //     return ret;
-
-    // }
-
 }

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
 Wed Nov 11 14:45:37 2015
@@ -229,8 +229,6 @@ class ServiceInstance
                try {
                        Process p = pb.start();
 
-            // TODO: we should attach these streams to readers in threads 
because too much output
-            //       can cause blocking, deadlock, ugliness.
                        InputStream stdout = p.getInputStream();
                        InputStream stderr = p.getErrorStream();
 
@@ -317,6 +315,9 @@ class ServiceInstance
         }
         logger.info(methodName, sset.getId(), "START INSTANCE COMPLETE");
 
+        stdout_lines.clear();
+        stderr_lines.clear();
+
         return numeric_id;
     }
 
@@ -355,58 +356,32 @@ class ServiceInstance
         ProcessBuilder pb = new ProcessBuilder(args);
         Map<String, String> env = pb.environment();
         env.put("DUCC_HOME", System.getProperty("DUCC_HOME"));
-            
-        ArrayList<String> stdout_lines = new ArrayList<String>();
-        ArrayList<String> stderr_lines = new ArrayList<String>();
+
+        StdioListener sin_listener = null;
+        StdioListener ser_listener = null;
+
         int rc = 0;
         try {
             Process p = pb.start();
-                
+
+            InputStream stdout = p.getInputStream();
+                       InputStream stderr = p.getErrorStream();
+
+            sin_listener = new StdioListener(1, stdout, true);
+            ser_listener = new StdioListener(2, stderr, true);
+            Thread sol = new Thread(sin_listener);
+            Thread sel = new Thread(ser_listener);
+            sol.start();
+            sel.start();
+   
             rc = p.waitFor();
             logger.info(methodName, sset.getId(), "DuccServiceCancel returns 
with rc", rc);
 
-            if (logger.isTrace() || (rc != 0)) {
-                InputStream stdout = p.getInputStream();
-                InputStream stderr = p.getErrorStream();
-                BufferedReader stdout_reader = new BufferedReader(new 
InputStreamReader(stdout));
-                BufferedReader stderr_reader = new BufferedReader(new 
InputStreamReader(stderr));
-                
-                String line = null;
-                while ( (line = stdout_reader.readLine()) != null ) {
-                    stdout_lines.add(line);
-                }
-                
-                line = null;
-                while ( (line = stderr_reader.readLine()) != null ) {
-                    stderr_lines.add(line);
-                }
-            }
-                
+            sin_listener.stop();
+            ser_listener.stop();                
         } catch (Throwable t) {
-            // TODO Auto-generated catch block
             logger.error(methodName, null, t);
         }
-
-        if ( logger.isTrace() || ( rc != 0) ) {
-            boolean inhibit_cp = false;
-            for ( String s : stdout_lines ) {
-                // simple logic to inhibit printing the danged classpath
-                if ( inhibit_cp ) {
-                    inhibit_cp = false;
-                    logger.info(methodName, sset.getId(), "Instance", 
numeric_id, "<INHIBITED CP>");
-                } else {
-                    logger.info(methodName, sset.getId(), "Instance", 
numeric_id, "Stop stdout:", s);
-                }
-                
-                if ( s.indexOf("-cp") >= 0 ) {
-                    inhibit_cp = true;
-                }
-            }
-            
-            for ( String s : stderr_lines ) {
-                logger.info(methodName, sset.getId(), "Instance", numeric_id, 
"Stop stderr:", s);
-            }
-        }
     }
 
     class StdioListener
@@ -416,8 +391,9 @@ class ServiceInstance
         String tag;
         boolean done = false;
         int which = 0;
+        boolean ignore = false;
 
-        StdioListener(int which, InputStream in)
+        StdioListener(int which, InputStream in, boolean ignore)
         {
             this.in = in;
             this.which = which;
@@ -425,6 +401,13 @@ class ServiceInstance
                case 1: tag = "STDOUT: "; break;
                case 2: tag = "STDERR: "; break;
             }
+            this.ignore = ignore;
+            this.ignore = ignore;
+        }
+
+        StdioListener(int which, InputStream in)
+        {
+            this(which, in, false);
         }
 
         void stop()
@@ -450,6 +433,8 @@ class ServiceInstance
                         logger.info(methodName, sset.getId(), msg);
                         return;
                     }
+                    if ( ignore ) continue;  // just discarding it
+
                     switch ( which ) {
                         case 1:
                             stdout_lines.add(s);

Modified: 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
URL: 
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
 (original)
+++ 
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
 Wed Nov 11 14:45:37 2015
@@ -18,6 +18,8 @@
 */
 package org.apache.uima.ducc.sm;
 
+import java.io.File;
+import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.sql.Timestamp;
@@ -32,6 +34,8 @@ import java.util.Timer;
 import java.util.TimerTask;
 import java.util.TreeMap;
 
+import javax.naming.ServiceUnavailableException;
+
 import org.apache.uima.UIMAFramework;
 import org.apache.uima.ducc.cli.IUiOptions.UiOption;
 import org.apache.uima.ducc.cli.UimaAsPing;
@@ -1048,8 +1052,14 @@ public class ServiceSet
             monitor.init(null);
             monitor.clearQueues();
             monitor.stop(      );
+        } catch (IOException e) {
+            Throwable t = e.getCause();
+            // ServiceUnavailbleException means it's gone already and we're 
processing some sort of stale state
+            // that's still floating around.
+            if ( ! (t instanceof ServiceUnavailableException) ) {
+                logger.info(methodName, id, e);
+            }
         } catch (Throwable e) {
-            // totally not a problem, just lost it
             logger.info(methodName, id, e.toString());
         }
     }
@@ -1866,6 +1876,8 @@ public class ServiceSet
         };
 
         ProcessBuilder pb = new ProcessBuilder(args);
+        pb.redirectOutput(new File("/dev/null"));
+        pb.redirectError(new File("/dev/null"));
         try {
             Process p = pb.start();
             int rc = p.waitFor();


Reply via email to