Author: challngr
Date: Tue Sep 22 18:10:57 2015
New Revision: 1704682
URL: http://svn.apache.org/viewvc?rev=1704682&view=rev
Log:
UIMA-4577 Command-line database loader.
Added:
uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader (with props)
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/database.py
uima/sandbox/uima-ducc/trunk/src/main/admin/db.py
uima/sandbox/uima-ducc/trunk/src/main/admin/db_console
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbObject.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ProcessToJobList.java
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/database.py
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/database.py?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/database.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/database.py Tue Sep 22 18:10:57
2015
@@ -21,7 +21,6 @@
# This is common code to deal wiht the orientdb configuration xml
-
global domloaded
try:
from xml.dom.minidom import parse
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db.py
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db.py?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db.py Tue Sep 22 18:10:57 2015
@@ -31,6 +31,8 @@ from ducc_util import DuccUtil
from properties import Properties
import database as db
+# This presents an interactive 'main' for performing various tasks with the
database
+
class DbUtil(DuccUtil):
def __init__(self):
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/db_console
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_console?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_console (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_console Tue Sep 22 18:10:57
2015
@@ -32,6 +32,8 @@ class DbConsole(DuccUtil):
DuccUtil.__init__(self)
def main(self, rgv):
+ print 'This utility is deprecated. Use db.py (no parameters)'
+ sys.exit(1)
# The dom gets loaded by the import above - or not
if ( not db.domloaded ):
print "Unable to read database configuration; insure the installed
Python supports xml.dom.minidom"
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader?rev=1704682&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader Tue Sep 22 18:10:57
2015
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+
+import os
+import sys
+import getopt
+
+
+#
+# This is a one-time-use utility to move your history and checkpoint files
+# int the database. It will not run if the database is already populated.
+#
+# If you need to run it again, you must first remove or drop your existing
database.
+#
+from ducc_util import DuccUtil
+
+class DbConvert(DuccUtil):
+
+ def __init__(self):
+ DuccUtil.__init__(self)
+
+ def usage(self, *args):
+ if ( args[0] != None ):
+ print ''
+ print ' '.join(args)
+ print ''
+ print 'Usage:'
+ print ''
+ print 'db_loader -i in -o out [--no-archive]'
+ print ''
+ print 'Where:'
+ print ' -i in'
+ print ' Specifies the DUCC_HOME of the DUCC runtime to be
moved into the database.'
+ print ' -o out'
+ print ' Specifies the DUCC_HOME of the DUCC runtime to contain
the database.'
+ print ' --noarchive'
+ print ' Suppresses archival of the input files.'
+ print ''
+ print 'Notes:'
+ print ' 1. The output location is not verfied as a DUCC runtime.
The database will be created'
+ print ' in that location nonetheless. After it is created it
may be manually moved to'
+ print ' the intended runtime.'
+ print ' 2. The "in" and "out" directories may be the same.'
+ print ' 3. Archival consists of renaming relevent input
directories and files by appending'
+ print ' ".archive" to their names. It is fully non-destructive
and may be reversed by'
+ print ' manually renaming them back to their original names.'
+ print ' 4. If a database already exists in the specified "out"
location, it is first dropped.'
+ print ' The schema is built and the database is fully
re-initialized.'
+
+ sys.exit(1);
+
+ def main(self, argv):
+
+ in_home = None
+ out_home = None
+ archive = True
+ try:
+ opts, args = getopt.getopt(argv, 'i:o:h?', ['no-archive'])
+ except:
+ self.usage("Invalid arguments", ' '.join(argv))
+
+ for ( o, a ) in opts:
+ if o in ('-i'):
+ in_home = a
+ elif o in ('-o'):
+ out_home = a
+ elif o in ('--no-archive'):
+ archive = False
+ else:
+ self.usage(None)
+
+
+ if ( in_home == None ):
+ self.usage("Missing input DUCC_HOME")
+ if ( out_home == None ):
+ self.usage("Missing output DUCC_HOME");
+
+ # DDUCC_HOME=/home/challngr/ducc_runtime_db -Xmx4G
-Dstorage.useWAL=true -Dtx.useLog=false -Dstorage.wal.syncOnPageFlush=false
+ DUCC_JVM_OPTS = ''
+ DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -DDUCC_HOME=' + self.DUCC_HOME
+ DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -Xmx4G'
+ if ( not archive ):
+ DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -DDONT_ARCHIVE'
+
+ CMD = [self.java(), DUCC_JVM_OPTS,
'org.apache.uima.ducc.database.DbLoader', in_home, out_home]
+ CMD = ' '.join(CMD)
+
+ os.environ['CLASSPATH'] = os.environ['CLASSPATH'] + ':' +
self.DUCC_HOME + "/lib/orientdb/*"
+ os.environ['CLASSPATH'] = os.environ['CLASSPATH'] + ':' +
self.DUCC_HOME + "/resources"
+ print os.environ['CLASSPATH']
+ os.system(CMD)
+
+
+if __name__ == "__main__":
+ console = DbConvert()
+ console.main(sys.argv[1:])
+
+
Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/db_loader
------------------------------------------------------------------------------
svn:executable = *
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Tue Sep 22
18:10:57 2015
@@ -41,7 +41,7 @@ import database as db
class PostInstall():
- def usage(self, *msg):
+ def usage(self, msg):
if ( msg != None ):
print ' '.join(msg)
@@ -61,7 +61,7 @@ class PostInstall():
print " [-j, --jvm] <path to java executable>"
print " This is the full path to java command to be used to
start DUCC; e.g., /usr/bin/java"
print ""
- print " [-d --db-password] <root password for database>"
+ print " [-, --db-password] <root password for database>"
print " This is the password DUCC uses to manage the database."
print ""
print " [-h, -? --help]"
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbConstants.java
Tue Sep 22 18:10:57 2015
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+
package org.apache.uima.ducc.database;
/**
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
Tue Sep 22 18:10:57 2015
@@ -19,6 +19,7 @@
package org.apache.uima.ducc.database;
+import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.database.DbConstants.DbEdge;
import org.apache.uima.ducc.database.DbConstants.DbVertex;
import org.apache.uima.ducc.database.DbConstants.DuccVertexBase;
@@ -34,6 +35,7 @@ import com.tinkerpop.blueprints.impls.or
public class DbCreate
{
+ DuccLogger logger = null;
String dburl;
String adminid = "root";
String adminpw = null;
@@ -45,6 +47,13 @@ public class DbCreate
this.dburl = dburl;
}
+
+ public DbCreate(String dburl, DuccLogger logger)
+ {
+ this.dburl = dburl;
+ this.logger = logger;
+ }
+
public DbCreate(String dburl, String adminid, String adminpw)
{
this.dburl = dburl;
@@ -52,22 +61,46 @@ public class DbCreate
this.adminpw = adminpw;
}
+ public void doLog(String methodName, Object ... msg)
+ {
+ if ( logger == null ) {
+
+ StringBuffer buf = new StringBuffer(methodName);
+ for ( Object o : msg ) {
+ buf.append(" ");
+ if ( o == null ) {
+ buf.append("<null>");
+ } else {
+ buf.append(o.toString());
+ }
+ }
+
+ System.out.println(buf);
+ } else {
+ logger.info(methodName, null, msg);
+ return;
+ }
+
+ }
+
void createEdgeType(OrientGraphNoTx g, DbEdge id)
{
+ String methodName = "createEdgeType";
String s = id.pname();
OrientEdgeType e = g.getEdgeType(s);
if ( e == null ) {
- System.out.println("Create edge " + s);
+ doLog(methodName, "Create edge", s);
g.createEdgeType(s);
}
}
void createVertexType(OrientGraphNoTx g, DbVertex id)
{
+ String methodName = "createVertexType";
String s = id.pname();
OrientVertexType e = g.getVertexType(s);
if ( e == null ) {
- System.out.println("Create vertex " + s);
+ doLog(methodName, "Create vertex " + s);
e = g.createVertexType(s, DuccVertexBase.VBase.pname());
}
}
@@ -80,7 +113,7 @@ public class DbCreate
String base = DuccVertexBase.VBase.pname();
OrientVertexType e = g.getVertexType(base);
if ( e == null ) {
- System.out.println("Create base vertex class " + base);
+ doLog(methodName, "Create base vertex class " + base);
e = g.createVertexType(base);
OProperty p = e.createProperty(DbConstants.DUCCID, OType.LONG);
p.setMandatory(true);
@@ -89,11 +122,11 @@ public class DbCreate
String sql = "create index i_ducc_dbid on " + base + "(" +
DbConstants.DUCCID + ") notunique";
g.command(new OCommandSQL(sql)).execute();
- System.out.println("(sql)Created index i_ducc_dbid on class " +
base + " for " + DbConstants.DUCCID);
+ doLog(methodName, "(sql)Created index i_ducc_dbid on class " +
base + " for " + DbConstants.DUCCID);
sql = "create index i_ducc_dbcat on " + base + "(" +
DbConstants.DUCC_DBCAT + ") notunique";
g.command(new OCommandSQL(sql)).execute();
- System.out.println("(sql)Created index i_ducc_dbcat on class " +
base + " for " + DbConstants.DUCC_DBCAT);
+ doLog(methodName, "(sql)Created index i_ducc_dbcat on class " +
base + " for " + DbConstants.DUCC_DBCAT);
}
@@ -130,6 +163,8 @@ public class DbCreate
boolean createDatabase()
throws Exception
{
+ String methodName = "createDatabase";
+
if ( adminpw == null ) {
adminpw = DbManager.dbPassword();
}
@@ -139,16 +174,16 @@ public class DbCreate
admin.connect(adminid, adminpw); // connect to the
server
if ( ! admin.existsDatabase("plocal") ) {
- System.out.println("Database " + dburl + " does not exist,
attempting to create it.");
+ doLog(methodName, "Database " + dburl + " does not exist,
attempting to create it.");
admin.createDatabase("graph", "plocal");
if ( ! admin.existsDatabase() ) {
- System.out.println("Cannot create database " + dburl);
+ doLog(methodName, "Cannot create database " + dburl);
return false;
}
factory = new OrientGraphFactory(dburl);
if ( factory == null ) {
- System.out.println("Cannot create graph factory for " +
dburl);
+ doLog(methodName, "Cannot create graph factory for " +
dburl);
return false;
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
Tue Sep 22 18:10:57 2015
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
package org.apache.uima.ducc.database;
import java.util.ArrayList;
@@ -25,14 +43,20 @@ import com.tinkerpop.blueprints.impls.or
public class DbHandle
{
- //private DuccLogger logger = DuccLogger.getLogger(DbHandle.class, "DB");
// get the component logger
- private DuccLogger logger =
DuccService.getDuccLogger(DbHandle.class.getName());
+ private DuccLogger logger = null;
DbManager manager;
public OrientBaseGraph graphDb;
DbHandle(DbManager manager, OrientBaseGraph graphDb)
{
+ if ( DuccService.getDuccLogger() == null ) {
+ // not running within a ducc service - just get a regular logger
+ logger = DuccLogger.getLogger(DbHandle.class, "DB");
+ } else {
+ // running within a ducc service - get the component logger
+ logger = DuccService.getDuccLogger(DbHandle.class.getName());
+ }
this.manager = manager;
this.graphDb = graphDb;
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
Tue Sep 22 18:10:57 2015
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
package org.apache.uima.ducc.database;
import java.io.File;
@@ -16,6 +34,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.uima.ducc.common.Pair;
import org.apache.uima.ducc.common.persistence.services.IStateServices;
import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.Utils;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.database.DbConstants.DbCategory;
import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
@@ -32,41 +51,109 @@ import com.orientechnologies.orient.core
public class DbLoader
{
DuccLogger logger = DuccLogger.getLogger(DbLoader.class, "DBLOAD");
+ String DUCC_HOME;
+ boolean archive = true; // for debug and test, bypass archive-rename
HistoryManagerDb hmd = null;
StateServicesDb ssd = null;
// String history_url = "remote:localhost/DuccHistory";
// String state_url =
"plocal:/home/challngr/ducc_runtime_db/database/databases/DuccHistoryT";
- String state_url = "plocal:/users/challngr/DuccHistoryT";
+ String state_url = null;
// String jobHistory = System.getProperty("user.home") +
"/ducc_runtime_db/history/jobs";
- String jobHistory = "/home/ducc/ducc_runtime/history/jobs";
+ String jobHistory = "/history/jobs";
// String reservationHistory = System.getProperty("user.home") +
"/ducc_runtime/history/reservations";
- String reservationHistory = "/home/ducc/ducc_runtime/history/reservations";
+ String reservationHistory = "/history/reservations";
//String serviceHistory = System.getProperty("user.home") +
"/ducc_runtime/history/services";
- String serviceHistory = "/home/ducc/ducc_runtime/history/services";
+ String serviceHistory = "/history/services";
//String serviceHistory = System.getProperty("user.home") +
"/ducc_runtime/history/services";
- String serviceRegistryHistory =
"/home/ducc/ducc_runtime/history/services-registry";
+ String serviceRegistryHistory = "/history/services-registry";
//String serviceRegistry = System.getProperty("user.home") +
"/ducc_runtime/state/services";
- String serviceRegistry = "/home/ducc/ducc_runtime/state/services";
+ String serviceRegistry = "/state/services";
- String checkpointFile = "/home/ducc/ducc_runtime/state/orchestrator.ckpt";
+ String checkpointFile = "/state/orchestrator.ckpt";
String archive_key = IStateServices.archive_key;
String archive_flag = IStateServices.archive_flag;
int nthreads = 40;
AtomicInteger counter = new AtomicInteger(0);
- public DbLoader()
+ int joblimit = Integer.MAX_VALUE;
+ int reservationlimit = Integer.MAX_VALUE;
+ int servicelimit = Integer.MAX_VALUE;
+ int registrylimit = Integer.MAX_VALUE;
+
+ boolean dojobs = true;
+ boolean doreservations = true;
+ boolean doservices = true;
+ boolean doregistry = true;
+ boolean docheckpoint = true;
+
+ public DbLoader(String from, String to)
throws Exception
{
- // System.setProperty("ducc.history.database.url", history_url);
+ String methodName = "<ctr>";
+ DUCC_HOME = System.getProperty("DUCC_HOME");
+ if ( DUCC_HOME == null ) {
+ System.out.println("System proprety -DDUCC_HOME must be set.");
+ System.exit(1);
+ }
+
+ if ( System.getProperty("DONT_ARCHIVE") != null ) archive = false;
+
+ File f = new File(from);
+ if ( ! f.isDirectory() ) {
+ System.out.println("'from' must be a directory");
+ System.exit(1);
+ }
+
+ f = new File(to);
+ if ( ! f.isDirectory() ) {
+ System.out.println("'to' must be a directory");
+ System.exit(1);
+ }
+
+ String databasedir = to + "/database/databases";
+ String databasename = databasedir + "/DuccState";
+ // We always use a non-networked version for loading
+ state_url = "plocal:" + databasedir + "/DuccState";
System.setProperty("ducc.state.database.url", state_url);
+
+ f = new File(databasedir);
+ if ( f.exists() ) {
+ f = new File(databasename);
+ if ( f.exists() ) {
+ logger.info(methodName, null, "Dropping existing database.");
+ DbManager dbm = new DbManager(state_url, logger);
+ dbm.init();
+ dbm.drop();
+ dbm.shutdown();
+ }
+ } else {
+ try {
+ if ( ! f.mkdirs() ) {
+ System.out.println("Cannot create database directory: " +
databasedir);
+ System.exit(1);
+ }
+ System.out.println("Created database directory " +
databasedir);
+ } catch ( Exception e ) {
+ System.out.println("Cannot create database directory: " +
databasedir + ":" + e.toString());
+ System.exit(1);
+ }
+ }
+
+
+ jobHistory = from + jobHistory;
+ reservationHistory = from + reservationHistory;
+ serviceHistory = from + serviceHistory;
+ serviceRegistryHistory = from + serviceRegistryHistory;
+ serviceRegistry = from + serviceRegistry;
+ checkpointFile = from + checkpointFile;
}
void closeStream(InputStream in)
@@ -77,11 +164,27 @@ public class DbLoader
public void loadJobs()
{
String methodName = "loadJobs";
+
+ File dir = new File(jobHistory);
+ if ( !dir.isDirectory() ) {
+ logger.info(methodName, null, "Cannot find job history; skipping
load of jobs.");
+ return;
+ }
+
+ File[] files = dir.listFiles();
+ if ( files.length == 0 ) {
+ logger.info(methodName, null, "No jobs to move to database.");
+ return;
+ }
+ logger.info(methodName, null, "Reading", files.length, "jobs.");
+
LinkedBlockingQueue<File> queue = new LinkedBlockingQueue<File>();
- // int max_to_load = Integer.MAX_VALUE; // or Integer.MAX_VALUE for
'all of them'
- int max_to_load = 1000; // or Integer.MAX_VALUE for 'all of them'
+ int max_to_load = joblimit;
+ // int max_to_load = 1000; // or Integer.MAX_VALUE for 'all of them'
int nth = Math.min(nthreads, max_to_load);
+ nth = Math.min(nth, files.length);
+
JobLoader[] loader = new JobLoader[nth];
Thread[] threads = new Thread[nth];
List<Long> ids = new ArrayList<Long>();
@@ -92,9 +195,6 @@ public class DbLoader
threads[i].start();
}
- File dir = new File(jobHistory);
- File[] files = dir.listFiles();
- logger.info(methodName, null, "Reading", files.length, "jobs.");
int c = 0;
for ( File f : files) {
@@ -128,16 +228,35 @@ public class DbLoader
try { threads[i].join(); } catch ( InterruptedException e ) {}
}
+ if ( archive ) {
+ File renameTo = new File(dir + ".archive");
+ dir.renameTo(renameTo);
+ }
}
public void loadReservations()
{
String methodName = "loadReservations";
+
+ File dir = new File(reservationHistory);
+ if ( ! dir.isDirectory() ) {
+ logger.info(methodName, null, "No reservation directory found;
skipping database load of reservations.");
+ return;
+ }
+
+ File[] files = dir.listFiles();
+ if ( files.length == 0 ) {
+ logger.info(methodName, null, "No reservation history files to
convert.");
+ return;
+ }
+ logger.info(methodName, null, "Reading", files.length, "reservation
instances.");
+
LinkedBlockingQueue<File> queue = new LinkedBlockingQueue<File>();
- //int max_to_load = Integer.MAX_VALUE;
- int max_to_load = 1000;
+ int max_to_load = reservationlimit;
+ //int max_to_load = 1000;
int nth = Math.min(nthreads, max_to_load);
+ nth = Math.min(nth, files.length);
ReservationLoader[] loader = new ReservationLoader[nth];
Thread[] threads = new Thread[nth];
ArrayList<Long> ids = new ArrayList<Long>();
@@ -146,11 +265,7 @@ public class DbLoader
loader[i] = new ReservationLoader(queue, ids);
threads[i] = new Thread(loader[i]);
threads[i].start();
- }
-
- File dir = new File(reservationHistory);
- File[] files = dir.listFiles();
- logger.info(methodName, null, "Reading", files.length, "reservation
instances.");
+ }
int c = 0;
for ( File f : files ) {
@@ -185,17 +300,37 @@ public class DbLoader
logger.info(methodName, null, "Joining thread (reservations).", i);
try { threads[i].join(); } catch ( InterruptedException e ) {}
}
+
+ if ( archive ) {
+ File renameTo = new File(dir + ".archive");
+ dir.renameTo(renameTo);
+ }
}
public void loadServices()
{
String methodName = "loadServices";
+ File dir = new File(serviceHistory);
+ if ( ! dir.isDirectory() ) {
+ logger.info(methodName, null, "No service history directory found;
skipping load of service history.");
+ return;
+ }
+
+ File[] files = dir.listFiles();
+
+ if ( files.length == 0 ) {
+ logger.info(methodName, null, "No service history files to
convert.");
+ return;
+ }
+ logger.info(methodName, null, "Reading", files.length, "service
instances.");
+
LinkedBlockingQueue<File> queue = new LinkedBlockingQueue<File>();
- // int max_to_load = Integer.MAX_VALUE;
- int max_to_load = 1000;
+ int max_to_load = servicelimit;
+ // int max_to_load = 1000;
int nth = Math.min(nthreads, max_to_load);
+ nth = Math.min(nth, files.length);
ServiceLoader[] loader = new ServiceLoader[nth];
Thread[] threads = new Thread[nth];
ArrayList<Long> ids = new ArrayList<Long>();
@@ -206,10 +341,6 @@ public class DbLoader
threads[i].start();
}
- File dir = new File(serviceHistory);
- File[] files = dir.listFiles();
- logger.info(methodName, null, "Reading", files.length, "service
instances.");
-
int c = 0;
for ( File f : files ) {
String s = f.toString();
@@ -243,16 +374,34 @@ public class DbLoader
try { threads[i].join(); } catch ( InterruptedException e ) {}
}
+ if ( archive ) {
+ File renameTo = new File(dir + ".archive");
+ dir.renameTo(renameTo);
+ }
}
public void loadServiceRegistry(String registry, boolean isHistory)
{
String methodName = "loadServiceRegistry";
+ int c = 0;
+ File dir = new File(registry);
+ File[] files = dir.listFiles();
+ if ( files.length == 0 ) {
+ if ( isHistory ) {
+ logger.info(methodName, null, "Nothing in service registry
history to move to database");
+ } else {
+ logger.info(methodName, null, "Nothing in service registry to
move to database");
+ }
+ return;
+ }
+
LinkedBlockingQueue<Pair<String, Boolean>> queue = new
LinkedBlockingQueue<Pair<String, Boolean>>();
- int max_to_load = Integer.MAX_VALUE;
+ int max_to_load = registrylimit;
int nth = Math.min(nthreads, max_to_load);
+ nth = Math.min(nth, files.length);
+
ServiceRegistrationLoader[] loader = new
ServiceRegistrationLoader[nth];
Thread[] threads = new Thread[nth];
ArrayList<Long> ids = new ArrayList<Long>();
@@ -263,9 +412,6 @@ public class DbLoader
threads[i].start();
}
- int c = 0;
- File dir = new File(registry);
- File[] files = dir.listFiles();
logger.info(methodName, null, "Reading", files.length, "service files
(2 per instance).");
for ( File f : files ) {
String s = f.toString();
@@ -298,14 +444,29 @@ public class DbLoader
try { threads[i].join(); } catch ( InterruptedException e ) {}
}
+ if ( archive ) {
+ File renameTo = new File(dir + ".archive");
+ dir.renameTo(renameTo);
+ }
}
void loadCheckpoint()
throws Exception
{
String methodName = "loadCheckpoint";
-
- //Checkpointable obj = null;
+
+ File f = new File(checkpointFile);
+ if ( ! f.exists() ) {
+ logger.info(methodName, null, "No checkpoint file to convert.");
+ return;
+ }
+
+ //
+ // A note - the Checkpointable object might be in the "wrong" package
and can't be
+ // cast properly. When putting it into database we have to
pick out the
+ // fields anyway. So here we use introspection to get the
fields and
+ // create the database entries.
+ //
FileInputStream fis = null;
ObjectInputStream in = null;
try {
@@ -331,7 +492,11 @@ public class DbLoader
fis.close();
in.close();
}
-
+
+ if ( archive ) {
+ File renameTo = new File(f + ".archive");
+ f.renameTo(renameTo);
+ }
}
void run()
@@ -339,7 +504,7 @@ public class DbLoader
{
String methodName = "run";
- DbCreate cr = new DbCreate(state_url);
+ DbCreate cr = new DbCreate(state_url, logger);
cr.createPlocalDatabase();
logger.info(methodName, null, "storage.useWAL",
System.getProperty("storage.useWAL"));
@@ -351,7 +516,7 @@ public class DbLoader
hmd = new HistoryManagerDb(logger);
- if ( true ) loadCheckpoint();
+ if ( docheckpoint ) loadCheckpoint();
OGlobalConfiguration.USE_WAL.setValue(false);
@@ -359,32 +524,34 @@ public class DbLoader
// ---------- Load job history
- if ( true ) loadJobs();
+ if ( dojobs ) loadJobs();
// ---------- Load reservation history
- if ( true ) loadReservations();
+ if ( doreservations ) loadReservations();
// ---------- Load service isntance and AP history
- if ( true ) loadServices();
+ if ( doservices ) loadServices();
// ---------- Load service registry
- ssd = new StateServicesDb();
- ssd.init(logger);
- loadServiceRegistry(serviceRegistry, false);
- try {
- ssd.shutdown();
- } catch ( Exception e ) {
- e.printStackTrace();
+ if ( doregistry ) {
+ ssd = new StateServicesDb();
+ ssd.init(logger);
+ if ( false ) loadServiceRegistry(serviceRegistry, false);
+ try {
+ ssd.shutdown();
+ } catch ( Exception e ) {
+ e.printStackTrace();
+ }
+
+ // ---------- Load service registry history
+ ssd = new StateServicesDb();
+ ssd.init(logger);
+ if ( false ) loadServiceRegistry(serviceRegistryHistory,
true);
}
- // ---------- Load service registry history
- ssd = new StateServicesDb();
- ssd.init(logger);
- loadServiceRegistry(serviceRegistryHistory, true);
-
OGlobalConfiguration.USE_WAL.setValue(true);
- if ( true ) loadCheckpoint();
+ if ( docheckpoint ) loadCheckpoint();
@@ -403,9 +570,23 @@ public class DbLoader
public static void main(String[] args)
{
+ if ( args.length != 2 ) {
+ System.out.println("USage: DbLoader from to");
+ System.out.println("");
+ System.out.println("Where:");
+ System.out.println(" from");
+ System.out.println(" is the DUCC_HOME you wish to convert");
+ System.out.println(" to");
+ System.out.println(" is the DUCC_HOME contining the new
database");
+ System.out.println("");
+ System.out.println("'from' and 'to' may be the same thing");
+ System.exit(1);
+ }
+
+
DbLoader dbl = null;
try {
- dbl = new DbLoader();
+ dbl = new DbLoader(args[0], args[1]);
dbl.run();
} catch ( Exception e ) {
e.printStackTrace();
@@ -441,13 +622,14 @@ public class DbLoader
in = new ObjectInputStream(fis);
job = (IDuccWorkJob) in.readObject();
logger.info(methodName, job.getDuccId(), "Time to read
job:", System.currentTimeMillis() - now);
+ hmd.saveJobUnsafe(job);
} catch(Exception e) {
logger.info(methodName, null, e);
} finally {
closeStream(in);
closeStream(fis);
+ counter.getAndDecrement();
}
- hmd.saveJobUnsafe(job);
} catch ( InterruptedException e ) {
return;
@@ -456,9 +638,10 @@ public class DbLoader
}
synchronized(ids) {
- ids.add(job.getDuccId().getFriendly());
+ if ( job != null ) {
+ ids.add(job.getDuccId().getFriendly());
+ }
}
- counter.getAndDecrement();
}
}
}
@@ -492,24 +675,28 @@ public class DbLoader
in = new ObjectInputStream(fis);
svc = (IDuccWorkService) in.readObject();
logger.info(methodName, svc.getDuccId(), "Time to read
service:", System.currentTimeMillis() - now);
+ hmd.saveServiceUnsafe(svc);
} catch(Exception e) {
+ logger.info(methodName, null, "Error reading or saving
service:", f);
logger.info(methodName, null, e);
} finally {
closeStream(in);
closeStream(fis);
+ counter.getAndDecrement();
}
- hmd.saveServiceUnsafe(svc);
} catch ( InterruptedException e ) {
return;
} catch ( Exception e ){
+ logger.info(methodName, null, "Error reading or saving
service:", f);
logger.info(methodName, null, e);
}
synchronized(ids) {
- ids.add(svc.getDuccId().getFriendly());
+ if ( svc != null ) {
+ ids.add(svc.getDuccId().getFriendly());
+ }
}
- counter.getAndDecrement();
}
}
}
@@ -542,13 +729,15 @@ public class DbLoader
in = new ObjectInputStream(fis);
res = (IDuccWorkReservation) in.readObject();
logger.info(methodName, res.getDuccId(), "Time to read
reservation:", System.currentTimeMillis() - now);
+ hmd.saveReservationUnsafe(res);
} catch(Exception e) {
logger.info(methodName, null, e);
} finally {
closeStream(in);
closeStream(fis);
+ counter.getAndDecrement();
}
- hmd.saveReservationUnsafe(res);
+
} catch ( InterruptedException e ) {
return;
@@ -557,9 +746,10 @@ public class DbLoader
}
synchronized(ids) {
- ids.add(res.getDuccId().getFriendly());
+ if ( res != null ) {
+ ids.add(res.getDuccId().getFriendly());
+ }
}
- counter.getAndDecrement();
}
}
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
Tue Sep 22 18:10:57 2015
@@ -94,6 +94,16 @@ public class DbManager
return ret;
}
+ public void drop()
+ throws Exception
+ {
+ OrientGraphNoTx graphDb = factory.getNoTx(); // the graph
instance
+ if ( graphDb == null ) {
+ throw new IllegalStateException("Cannot allocate graph instance
for " + dburl);
+ }
+ graphDb.drop();
+ }
+
public synchronized DbHandle open()
throws Exception
{
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbObject.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbObject.java?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbObject.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbObject.java
Tue Sep 22 18:10:57 2015
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
package org.apache.uima.ducc.database;
import java.util.ArrayList;
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbTester.java
Tue Sep 22 18:10:57 2015
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
package org.apache.uima.ducc.database;
import java.io.File;
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ProcessToJobList.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ProcessToJobList.java?rev=1704682&r1=1704681&r2=1704682&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ProcessToJobList.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/ProcessToJobList.java
Tue Sep 22 18:10:57 2015
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
package org.apache.uima.ducc.database;
import java.util.ArrayList;
