Revision: 14628
http://gate.svn.sourceforge.net/gate/?rev=14628&view=rev
Author: ian_roberts
Date: 2011-12-01 17:08:07 +0000 (Thu, 01 Dec 2011)
Log Message:
-----------
Java reimplementation of the gcp.sh shell script in a hopefully more
platform-independent way. You can now use "java -jar gcp-cli.jar" instead of
"./gcp.sh".
Modified Paths:
--------------
gcp/trunk/.classpath
gcp/trunk/build.xml
Added Paths:
-----------
gcp/trunk/cli/
gcp/trunk/cli/build.xml
gcp/trunk/cli/src/
gcp/trunk/cli/src/gate/
gcp/trunk/cli/src/gate/cloud/
gcp/trunk/cli/src/gate/cloud/cli/
gcp/trunk/cli/src/gate/cloud/cli/Main.java
Property Changed:
----------------
gcp/trunk/
Property changes on: gcp/trunk
___________________________________________________________________
Modified: svn:ignore
- classes
gcp.jar
gcp-dist-*.zip
+ classes
gcp.jar
gcp-cli.jar
gcp-dist-*.zip
Modified: gcp/trunk/.classpath
===================================================================
--- gcp/trunk/.classpath 2011-12-01 16:24:58 UTC (rev 14627)
+++ gcp/trunk/.classpath 2011-12-01 17:08:07 UTC (rev 14628)
@@ -1,15 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
+ <classpathentry kind="src" output="cli/classes" path="cli/src"/>
<classpathentry kind="con"
path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="lib" path="conf"/>
<classpathentry kind="lib" path="lib/commons-httpclient-3.0.1.jar"/>
<classpathentry kind="lib"
path="lib/fastutil-5.0.3-heritrix-subset-1.0.jar"/>
<classpathentry kind="lib" path="lib/heritrix-1.14.4.jar"/>
- <classpathentry kind="lib" path="lib/mimir-client-3.2.0.jar"/>
<classpathentry kind="lib" path="lib/gate-6.1.jar"
sourcepath="/GATE/src"/>
<classpathentry kind="lib" path="lib/gate/log4j-1.2.14.jar"/>
<classpathentry kind="lib" path="lib/gate/ant.jar"/>
<classpathentry kind="lib" path="lib/gate/commons-io-1.4.jar"/>
+ <classpathentry kind="lib" path="lib/mimir-client-3.3.0.jar"/>
<classpathentry kind="output" path="classes"/>
</classpath>
Modified: gcp/trunk/build.xml
===================================================================
--- gcp/trunk/build.xml 2011-12-01 16:24:58 UTC (rev 14627)
+++ gcp/trunk/build.xml 2011-12-01 17:08:07 UTC (rev 14628)
@@ -11,6 +11,7 @@
<property name="src.dir" location="src" />
<property name="classes.dir" location="classes" />
+ <property name="cli.dir" location="cli" />
<property name="jar.location" location="gcp.jar" />
<!-- Path to compile - includes local libs and GATE ones -->
@@ -48,13 +49,19 @@
<!-- Clean up - remove .class and .jar files -->
<target name="clean" depends="clean.classes" >
<delete file="${jar.location}" />
+ <ant dir="${cli.dir}" target="clean" />
</target>
- <target name="distro" depends="clean,jar">
+ <target name="cli.jar">
+ <ant dir="${cli.dir}" target="jar" />
+ </target>
+
+ <target name="distro" depends="clean,jar,cli.jar">
<delete file="gcp-dist-${version}.zip" />
<zip destfile="gcp-dist-${version}.zip">
<zipfileset file="gcp.jar" prefix="gcp-${version}/lib"/>
<zipfileset file="dist/gcp.sh" filemode="755" prefix="gcp-${version}"/>
+ <zipfileset file="gcp-cli.jar" filemode="755" prefix="gcp-${version}"/>
<zipfileset dir="lib" includes="*.jar" prefix="gcp-${version}/lib"/>
<zipfileset dir="lib/gate" includes="*.jar" prefix="gcp-${version}/lib"/>
<zipfileset dir="conf" prefix="gcp-${version}/conf"/>
Property changes on: gcp/trunk/cli
___________________________________________________________________
Added: svn:ignore
+ classes
gcp-cli.jar
Added: gcp/trunk/cli/build.xml
===================================================================
--- gcp/trunk/cli/build.xml (rev 0)
+++ gcp/trunk/cli/build.xml 2011-12-01 17:08:07 UTC (rev 14628)
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<!-- $Id: build.xml 14422 2011-10-24 14:59:55Z ian_roberts $ -->
+<project name="GCP-cli" default="jar" basedir=".">
+ <property name="cli.src.dir" location="src" />
+ <property name="cli.classes.dir" location="classes" />
+ <property name="cli.jar.location" location="../gcp-cli.jar" />
+
+ <!-- create build directory structure -->
+ <target name="prepare">
+ <mkdir dir="${cli.classes.dir}" />
+ </target>
+
+ <!-- compile the source -->
+ <target name="compile" depends="prepare">
+ <javac srcdir="${cli.src.dir}"
+ destdir="${cli.classes.dir}"
+ debug="true"
+ debuglevel="lines,source"
+ source="1.5" target="1.5" />
+ </target>
+
+ <!-- create the JAR file -->
+ <target name="jar" depends="compile" >
+ <jar destfile="${cli.jar.location}"
+ update="false"
+ basedir="${cli.classes.dir}">
+ <manifest>
+ <attribute name="Main-Class" value="gate.cloud.cli.Main" />
+ </manifest>
+ </jar>
+ </target>
+
+ <!-- remove the generated .class files -->
+ <target name="clean.classes" >
+ <delete dir="${cli.classes.dir}" />
+ </target>
+
+ <!-- Clean up - remove .class and .jar files -->
+ <target name="clean" depends="clean.classes" >
+ <delete file="${cli.jar.location}" />
+ </target>
+
+</project>
Property changes on: gcp/trunk/cli/build.xml
___________________________________________________________________
Added: svn:eol-style
+ native
Added: gcp/trunk/cli/src/gate/cloud/cli/Main.java
===================================================================
--- gcp/trunk/cli/src/gate/cloud/cli/Main.java (rev 0)
+++ gcp/trunk/cli/src/gate/cloud/cli/Main.java 2011-12-01 17:08:07 UTC (rev
14628)
@@ -0,0 +1,316 @@
+package gate.cloud.cli;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.net.URL;
+import java.net.JarURLConnection;
+import java.util.Arrays;
+import java.util.List;
+import java.util.ArrayList;
+
+/**
+ * Command-line driver for GCP.
+ */
+public class Main {
+
+ protected static void printUsageMessage() {
+ System.out.println(
+ "Usage: java -jar gcp-cli.jar [optionalArguments] batch.xml\n" +
+ " java -jar gcp-cli.jar [optionalArguments] -d workingDir\n" +
+ "\n" +
+ "Optional arguments:\n" +
+ " -m maxMem : maximum Java heap size, passed as -Xmx (default
12G)\n" +
+ " -t threads : number of parallel processing threads (default 6)\n" +
+ " -Dname=value : Java system property settings, passed directly to the
VM\n" +
+ "\n" +
+ "If -d is specified, the following parameter is taken to refer to a
\"working\"\n" +
+ "directory. This directory is expected to contain a subdirectory named
\"in\".\n" +
+ "Batch specification files dropped in the in directory will be processed
one\n" +
+ "at a time, with the standard output and error piped to a file named for
the\n" +
+ "batch in a \"logs\" directory under the working directory. Successful
batches\n" +
+ "will be moved to a directory named \"out\" under the working
directory,\n" +
+ "batches that could not be processed are moved to a directory named
\"err\".\n" +
+ "Note that additional batches can be added to the input directory at any
time\n" +
+ "- whenever a batch finishes GCP will look in the input directory to see
if\n" +
+ "any more batches are available. If the directory contains a file
named\n" +
+ "\"shutdown.gcp\", GCP will shut down at the end of the currently
running\n" +
+ "batch, or immediately if there are no batches running.\n" +
+ "\n" +
+ "If -d is not specified, then the single non-optional argument is
interpreted\n" +
+ "as a batch file, which is executed, after which the process ends.\n" +
+ "\n" +
+ "This script respects the JAVA_OPTS environment variable, any VM
options\n" +
+ "placed in this variable will be passed to the Java VM.\n"
+ );
+ }
+
+ public static class TeeStreamGobbler extends Thread {
+ private PrintWriter pw = null;
+ private Process process;
+ public TeeStreamGobbler(Process p, File f) throws IOException {
+ this.process = p;
+ if(f != null) {
+ pw = new PrintWriter(new FileWriter(f));
+ }
+ }
+
+ public void run() {
+ InputStream in = process.getInputStream();
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ try {
+ String line = null;
+ while((line = br.readLine()) != null) {
+ if(pw != null) {
+ pw.println(line);
+ }
+ System.out.println(line);
+ }
+ } catch(IOException e) {
+ System.out.println("Exception reading from GCP process");
+ e.printStackTrace();
+ }
+ finally {
+ pw.close();
+ try {
+ br.close();
+ }
+ catch(IOException e) {
+ System.out.println("Exception closing GCP process input stream");
+ e.printStackTrace();
+ }
+ }
+
+ }
+ }
+
+ /**
+ * Install a signal handler that kills the spawned GCP process
+ * on CTRL-C, rather than killing this process. This is done using
+ * reflection so as not to compile-time depend on sun.misc. If
+ * anything goes wrong when installing the signal handler we warn
+ * and carry on without it.
+ */
+ protected static void installSignalHandler() {
+ try {
+ Class<?> signalClass = Class.forName("sun.misc.Signal");
+ Class<?> sigHandlerClass = Class.forName("sun.misc.SignalHandler");
+ Object sigInt =
signalClass.getConstructor(String.class).newInstance("INT");
+
+ class KillGcpHandler implements InvocationHandler {
+ public Object oldHandler;
+
+ public Object invoke(Object proxy, Method method, Object[] args)
+ throws Throwable {
+ if(method.getDeclaringClass() == Object.class) {
+ return method.invoke(this, args);
+ }
+ if("handle".equals(method.getName())) {
+ if(currentGcpProcess != null) {
+ System.out.println("Interrupting GCP process");
+ currentGcpProcess.destroy();
+ } else {
+ method.invoke(oldHandler, args);
+ }
+ }
+ return null;
+ }
+ }
+
+ KillGcpHandler handler = new KillGcpHandler();
+ Object handlerProxy =
Proxy.newProxyInstance(Main.class.getClassLoader(), new Class<?>[]
{sigHandlerClass}, handler);
+ handler.oldHandler = signalClass.getMethod("handle", signalClass,
sigHandlerClass).invoke(null, sigInt, handlerProxy);
+ }
+ catch(Exception e) {
+ System.out.println("Warning: couldn't install signal handler: " + e);
+ }
+ }
+
+ private static File gcpHome;
+
+ private static String maxMem = "12G";
+ private static String threads = "6";
+
+ // non-null only in dir mode
+ private static File workingDir = null;
+ // non-null only in file mode
+ private static File batchFile = null;
+
+ private static ProcessBuilder processBuilder = null;
+
+ private static Process currentGcpProcess = null;
+
+ private static List<String> javaOpts = new ArrayList<String>();
+
+ protected static void findGcpHome() throws Exception {
+ String gcpHomeFromSysprop = System.getProperty("gcp.home");
+ if(gcpHomeFromSysprop != null) {
+ gcpHome = new File(gcpHomeFromSysprop);
+ return;
+ }
+
+ String gcpHomeFromEnv = System.getenv("GCP_HOME");
+ if(gcpHomeFromEnv != null) {
+ gcpHome = new File(gcpHomeFromEnv);
+ return;
+ }
+
+ URL thisClassUrl = Main.class.getResource("Main.class");
+ if(thisClassUrl.getProtocol().equals("jar")) {
+ JarURLConnection conn = (JarURLConnection)thisClassUrl.openConnection();
+ String cliJar = conn.getJarFile().getName();
+ gcpHome = new File(cliJar).getParentFile();
+ return;
+ }
+
+ System.out.println("Could not determine location of GCP, please set
either");
+ System.out.println("- The system property gcp.home or");
+ System.out.println("- The environment variable GCP_HOME");
+ System.out.println("to point to the home directory of your GCP
installation");
+ System.exit(1);
+ }
+
+ protected static void parseArgs(String[] args) throws Exception {
+ int i = 0;
+ for(i = 0; i < args.length; i++) {
+ if(!args[i].startsWith("-")) break;
+ if("-m".equals(args[i])) {
+ maxMem = args[++i];
+ } else if("-t".equals(args[i])) {
+ threads = args[++i];
+ } else if("-d".equals(args[i])) {
+ workingDir = new File(args[++i]);
+ } else if(args[i].startsWith("-D")) {
+ javaOpts.add(args[i]);
+ } else {
+ System.out.println("Unrecognised option " + args[i]);
+ printUsageMessage();
+ System.exit(1);
+ }
+ }
+
+ if(workingDir == null) {
+ // not dir mode, first non-option argument is batch file
+ batchFile = new File(args[i]);
+ }
+ }
+
+ protected static void prepareProcessBuilder() {
+ List<String> cmdline = new ArrayList<String>();
+ cmdline.add(System.getProperty("java.home") + File.separator + "bin" +
File.separator + "java");
+ cmdline.add("-classpath");
+ StringBuilder classpath = new StringBuilder();
+ classpath.append(new File(gcpHome, "conf").getAbsolutePath());
+ classpath.append(File.pathSeparator);
+ classpath.append(new File(new File(gcpHome, "lib"),
"*").getAbsolutePath());
+ cmdline.add(classpath.toString());
+ cmdline.add("-Xmx" + maxMem);
+ cmdline.addAll(javaOpts);
+ cmdline.add("gate.cloud.batch.BatchRunner");
+ cmdline.add(threads);
+ // this will be replaced with the right batch file path at run time
+ cmdline.add("dummyBatchFileName.xml");
+ processBuilder = new ProcessBuilder(cmdline);
+ processBuilder.redirectErrorStream(true);
+ }
+
+ protected static int runGcp(File batchSpec, File outputFile) throws
Exception {
+ int returnCode = -1;
+ try {
+ List<String> cmdline = processBuilder.command();
+ cmdline.set(cmdline.size() - 1, batchSpec.getAbsolutePath());
+ currentGcpProcess = processBuilder.start();
+ new TeeStreamGobbler(currentGcpProcess, outputFile).start();
+ boolean gcpFinished = false;
+ while(!gcpFinished) {
+ try {
+ returnCode = currentGcpProcess.waitFor();
+ gcpFinished = true;
+ }
+ catch(InterruptedException e) {
+ // do nothing, wait again
+ }
+ }
+ } finally {
+ currentGcpProcess = null;
+ }
+ return returnCode;
+ }
+
+ public static void main(String[] args) throws Exception {
+ if(args.length == 0) {
+ printUsageMessage();
+ System.exit(1);
+ }
+ findGcpHome();
+ String javaOptsEnv = System.getenv("JAVA_OPTS");
+ if(javaOptsEnv != null) {
+ javaOpts.addAll(Arrays.asList(javaOptsEnv.split("\\s+")));
+ }
+ parseArgs(args);
+ prepareProcessBuilder();
+ installSignalHandler();
+
+ if(workingDir == null) {
+ if(!batchFile.canRead()) {
+ System.out.println("Specified batch file " + batchFile.getPath() + "
not found or not readable");
+ System.exit(1);
+ }
+ System.exit(runGcp(batchFile, null));
+ } else {
+ // dir mode
+ if(!workingDir.canRead()) {
+ System.out.println("Working directory " + workingDir.getPath() + " not
found or not readable");
+ System.exit(1);
+ }
+ File inDir = new File(workingDir, "in");
+ if(!inDir.canRead() || !inDir.canWrite()) {
+ System.out.println("Input directory " + inDir.getPath() + " not found
or not accessible");
+ System.exit(1);
+ }
+
+ File outDir = new File(workingDir, "out");
+ outDir.mkdirs();
+ File errDir = new File(workingDir, "err");
+ errDir.mkdirs();
+ File logDir = new File(workingDir, "logs");
+ logDir.mkdirs();
+
+ FilenameFilter xmlFilter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.endsWith(".xml");
+ }
+ };
+
+ File shutdownFile = new File(inDir, "shutdown.gcp");
+
+ while(!shutdownFile.exists()) {
+ String[] batchFiles = inDir.list(xmlFilter);
+ if(batchFiles.length == 0) {
+ // nothing to do, wait ten seconds and then try again
+ Thread.sleep(10000);
+ } else {
+ // find the first batch file in lexicographic order
+ Arrays.sort(batchFiles);
+ File theBatchFile = new File(inDir, batchFiles[0]);
+ File theLogFile = new File(logDir, batchFiles[0] + ".log");
+ // run GCP, move the batch file to out or err as appropriate
+ if(runGcp(theBatchFile, theLogFile) == 0) {
+ theBatchFile.renameTo(new File(outDir, batchFiles[0]));
+ } else {
+ theBatchFile.renameTo(new File(errDir, batchFiles[0]));
+ }
+ }
+ }
+ shutdownFile.delete();
+ }
+ }
+}
Property changes on: gcp/trunk/cli/src/gate/cloud/cli/Main.java
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
All the data continuously generated in your IT infrastructure
contains a definitive record of customers, application performance,
security threats, fraudulent activity, and more. Splunk takes this
data and makes sense of it. IT sense. And common sense.
http://p.sf.net/sfu/splunk-novd2d
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
