svn commit: r1724479 - /uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsServiceMonitor.java
Author: cwiklik Date: Wed Jan 13 18:46:20 2016 New Revision: 1724479 URL: http://svn.apache.org/viewvc?rev=1724479=rev Log: UIMA-4730 Modified to support new jmx query syntax in AMQ v 5.8.0 and beyond. It still supports older brokers as well Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsServiceMonitor.java Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsServiceMonitor.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsServiceMonitor.java?rev=1724479=1724478=1724479=diff == --- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsServiceMonitor.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsServiceMonitor.java Wed Jan 13 18:46:20 2016 @@ -20,6 +20,8 @@ package org.apache.uima.ducc.cli; import java.io.IOException; import java.text.DecimalFormat; +import java.util.Map; +import java.util.Map.Entry; import javax.management.MBeanServerConnection; import javax.management.MBeanServerInvocationHandler; @@ -67,13 +69,14 @@ public class UimaAsServiceMonitor String jmxFailure = null; -public UimaAsServiceMonitor(String qname, String broker_host, int broker_port) +String broker_host; +volatile boolean useNewJmxQueryString = false; +public UimaAsServiceMonitor(String qname, String broker_host, int broker_jmx_port) { -this.qname = qname; -this.broker_url = "service:jmx:rmi:///jndi/rmi://"+ broker_host + ":" + broker_port + "/jmxrmi"; +this.broker_host = broker_host; + this.qname = qname; +this.broker_url = "service:jmx:rmi:///jndi/rmi://"+ broker_host + ":" + broker_jmx_port + "/jmxrmi"; this.qstats = new ServiceStatistics(false, false, "N/A"); -// System.out.println("Broker url: " + broker_url); - } // public ServiceStatistics getStatistics() @@ -91,47 +94,76 @@ public class UimaAsServiceMonitor // return qstats; // } +private ObjectName getBrokerObjectName(MBeanServerConnection conn, String brokerQuery, String brokerNameProperty ) throws Exception { +ObjectName brokerObjectName = null; + for (Object nameObject : conn.queryNames(new ObjectName(brokerQuery), (QueryExp) null)) { +//find the brokername object +brokerObjectName = (ObjectName) nameObject; + +// the following code checks if the current object represents AMQ Broker MBean which looks like: +// org.apache.activemq:type=Broker,brokerName=localhost +// The above has just 2 properties: type and brokerName. We are after the brokerName here. Only +// one MBean has just these two properties. +Mapparts = brokerObjectName.getKeyPropertyList(); +boolean done = false; +if ( parts.size() == 2) { // looking for two properties Object only + for( Entry entry : parts.entrySet()) { +//if ( entry.getKey().equals("brokerName")) { // need this property +if ( entry.getKey().equals(brokerNameProperty)) { // need this property + done = true;// found a match. Got a broker ObjectName + break; +} + } +} +if ( done ) { + break; +} +//if (brokerObjectName.getCanonicalName().endsWith("Type=Broker")) { +// Extract just the name from the canonical name +//String brokerName = brokerObjectName.getCanonicalName().substring(0, brokerObjectName.getCanonicalName().indexOf(",")); +//System.out.println("Canonical name of broker is " + brokerObjectName.getCanonicalName()); +//System.out.println("broker name is " + brokerName); +} + return brokerObjectName; +} + /** * Connect to ActiveMq and find the mbean for the queue we're trying to monitor */ public void init(String parm /* parm not used in this impl */) throws Exception { -// String methodName = "init"; - JMXServiceURL url = new JMXServiceURL(broker_url); jmxc = JMXConnectorFactory.connect(url); MBeanServerConnection conn = jmxc.getMBeanServerConnection(); String jmxDomain = "org.apache.activemq"; - +String brokerJmxDomain = null; + // -// First get the broker name +// First get the broker name. Assume we are connecting to AMQ broker version >= 5.8.0 // -ObjectName brokerObjectName = null; -for (Object nameObject :
svn commit: r1724481 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server: DuccHandlerClassic.java DuccHandlerJsonFormat.java
Author: degenaro Date: Wed Jan 13 19:31:08 2016 New Revision: 1724481 URL: http://svn.apache.org/viewvc?rev=1724481=rev Log: UIMA-4741 DUCC Web Server (WS) when hovering over Done on jobs page (std) dev is always 0.0 Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java?rev=1724481=1724480=1724481=diff == --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java Wed Jan 13 19:31:08 2016 @@ -338,7 +338,7 @@ public class DuccHandlerClassic extends double min = Math.round(perWorkItemStatistics.getMin()/100.0)/10.0; double avg = Math.round(perWorkItemStatistics.getMean()/100.0)/10.0; double dev = Math.round(perWorkItemStatistics.getStandardDeviation()/100.0)/10.0; - done = ""+done+""; + done = ""+done+""; } sb.append(done); sb.append(""); Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java?rev=1724481=1724480=1724481=diff == --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java Wed Jan 13 19:31:08 2016 @@ -375,7 +375,7 @@ public class DuccHandlerJsonFormat exten double min = Math.round(perWorkItemStatistics.getMin()/100.0)/10.0; double avg = Math.round(perWorkItemStatistics.getMean()/100.0)/10.0; double dev = Math.round(perWorkItemStatistics.getStandardDeviation()/100.0)/10.0; - done = ""+done+""; + done = ""+done+""; } sb.append(done); sb.append("");
svn commit: r1724484 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd: fsm/wi/ActionEnd.java mh/MessageHandler.java
Author: degenaro Date: Wed Jan 13 19:37:02 2016 New Revision: 1724484 URL: http://svn.apache.org/viewvc?rev=1724484=rev Log: UIMA-4741 DUCC Web Server (WS) when hovering over Done on jobs page (std) dev is always 0.0 Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java?rev=1724484=1724483=1724484=diff == --- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java Wed Jan 13 19:37:02 2016 @@ -95,6 +95,7 @@ public class ActionEnd extends ActionEnd mb.append(Standardize.Label.avg.get()+wis.getMillisAvg()); mb.append(Standardize.Label.max.get()+wis.getMillisMax()); mb.append(Standardize.Label.min.get()+wis.getMillisMin()); + mb.append(Standardize.Label.stddev.get()+wis.getMillisStdDev()); logger.debug(location, ILogger.null_id, mb.toString()); } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java?rev=1724484=1724483=1724484=diff == --- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java Wed Jan 13 19:37:02 2016 @@ -165,6 +165,7 @@ public class MessageHandler implements I oi.setWorkItemFinishedMillisMin(wis.getMillisMin()); oi.setWorkItemFinishedMillisMax(wis.getMillisMax()); oi.setWorkItemFinishedMillisAvg(wis.getMillisAvg()); + oi.setWorkItemFinishedMillisStdDev(wis.getMillisStdDev()); oi.setWorkItemRunningMillisMin(rwis.getMillisMin()); oi.setWorkItemRunningMillisMax(rwis.getMillisMax()); oi.setWorkItemTodMostRecentStart(rwis.getTodMostRecentStart());
svn commit: r1724512 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4: admin/admin-commands.tex admin/ducc-database.tex admin/ducc-properties.tex ducc-aguide.tex insta
Author: challngr Date: Wed Jan 13 21:09:43 2016 New Revision: 1724512 URL: http://svn.apache.org/viewvc?rev=1724512=rev Log: UIMA-4745 Database updates to duccbookk. Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-database.tex Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/ducc-aguide.tex uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex?rev=1724512=1724511=1724512=diff == --- uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex Wed Jan 13 21:09:43 2016 @@ -35,6 +35,8 @@ The command \ducchome/admin/start\_ducc is used to start DUCC processes. If run with no parameters it takes the following actions: \begin{itemize} + \item Starts the ActiveMQ server. + \item Starts the database. \item Starts the management processes Resource Manager, Orchestrator, Process Manager, Services Manager, and Web Server on the local node (where start\_ducc is executed. \item Starts an agent process on every node named in the default node list. @@ -76,6 +78,8 @@ start\_ducc -c sm -c pm -c rm -c or@bj22 \item[sm]The Service Manager \item[ws]The Web Server \item[agent]Node Agents +\item[broker] ActiveMQ broker +\item[db] Database \end{description} \item[--nothreading] If specified, the command does not run in multi-threaded mode @@ -138,9 +142,16 @@ start\_ducc -c sm -c pm -c rm -c or@bj22 \label{subsec:admin.stop-ducc} \subsubsection{{\em Description:}} -Stop\_ducc is used to stop DUCC processes. If run with no parameters it takes the following -actions: -\todo Garbled by maven or docbook, update this +Stop\_ducc is used to stop DUCC processes. At least one parameter is required. +When {\em -a} is specified, the following actions are taken: +\begin{itemize} + \item Uses the ActiveMQ broker to broadcast a shutdown request to all +DUCC compoments, other than the ActiveMQ broker itself, and the database. + \item Waits a bit, for all daemons to stop. + \item Stops the database. + \item Stops the ActiveMQ broker. +\end{itemize} + \subsubsection{\em Usage:} @@ -202,10 +213,19 @@ start_ducc -c rm \item[pm] The Process Manager. \item[sm] The Service Manager. \item[ws] The Web Server. + \item[db] The database. \item[broker] The ActiveMQ broker (only if the broker is auto-managed). \item[agent\@node] Node Agent on the specified node. \end{description} + \item[-w, --wait {[time in seconds]}] If given, this signals the time to wait +after broadcasting the shutdown signal, and before stopping the ActiveMQ broker itself. +If not specified, the default is 60 seconds. + +NOTE: In production systems, it is generally wise to use the default of 60 seconds. For +test systems a shorter wait speeds cycle time. Be sure to use {\em check\_ducc -k} after +{\em stop\_ducc} if you change the wait time to insure all processes are actually stopped. + \item[--nothreading] If specified, the command does not run in multi-threaded mode even if it is supported on the local platform. @@ -595,3 +615,90 @@ Nodepool power \paragraph{Notes:} None. + +\subsection{db\_create} +\label{subsec:cli.db.create} + +\paragraph{Description:} +This command is used to initialize the database. Normally the database is initialized +during {\em ducc\_post\_install} but if this is an existing DUCC installation that is +being migrated from a version that does not use the database, it will be necessary to +initialize the database with this command. + +This command performs the following actions: +\begin{enumerate} + \item Starts the database. + \item Disables the default database superuser. + \item Installs a database superuser as ``ducc'' and sets the
svn commit: r1724509 - /uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java
Author: cwiklik Date: Wed Jan 13 20:33:05 2016 New Revision: 1724509 URL: http://svn.apache.org/viewvc?rev=1724509=rev Log: UIMA-4744 fixes RM admin channel not starting with the new version of Camel Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java?rev=1724509=1724508=1724509=diff == --- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java Wed Jan 13 20:33:05 2016 @@ -121,7 +121,8 @@ public class ResourceManagerComponent .process(new RmAdminEventProcessor(delegate)); } }); - + + getContext().startRoute("RMAdminRoute"); if (logger != null) { logger.info("startRMAdminChannel", null, "Admin Channel Activated on endpoint:" + endpoint); } @@ -203,7 +204,7 @@ public class ResourceManagerComponent String methodName = "start"; converter = new JobManagerConverter(scheduler, stabilityManager); -super.start(service, args); + super.start(service, args); DuccDaemonRuntimeProperties.getInstance().boot(DaemonName.ResourceManager, super.getProcessJmxUrl()); initStability = SystemPropertyResolver.getIntProperty("ducc.rm.init.stability", DEFAULT_INIT_STABILITY_COUNT); @@ -222,7 +223,7 @@ public class ResourceManagerComponent } else { startRmAdminChannel(adminEndpoint, this); } - + scheduler.init(); startStabilityTimer();
svn commit: r1724465 - in /uima/ruta/trunk: ruta-core/src/main/antlr3/org/apache/uima/ruta/parser/ ruta-core/src/main/java/org/apache/uima/ruta/ ruta-core/src/main/java/org/apache/uima/ruta/action/ ru
Author: pkluegl Date: Wed Jan 13 17:30:09 2016 New Revision: 1724465 URL: http://svn.apache.org/viewvc?rev=1724465=rev Log: UIMA-4735 - macros for actions and conditions Added: uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/action/MacroAction.java (with props) uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/condition/MacroCondition.java (with props) uima/ruta/trunk/ruta-core/src/test/java/org/apache/uima/ruta/condition/MacroConditionTest.java (with props) uima/ruta/trunk/ruta-ep-ide/src/main/java/org/apache/uima/ruta/ide/parser/ast/RutaMacroDeclaration.java (with props) Modified: uima/ruta/trunk/ruta-core/src/main/antlr3/org/apache/uima/ruta/parser/RutaParser.g uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/RutaEnvironment.java uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/RutaStream.java uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/action/ActionFactory.java uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/action/AssignAction.java uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/action/ImplicitVariableAssignmentAction.java uima/ruta/trunk/ruta-core/src/main/java/org/apache/uima/ruta/condition/ConditionFactory.java uima/ruta/trunk/ruta-core/src/test/java/org/apache/uima/ruta/action/MacroActionTest.java uima/ruta/trunk/ruta-docbook/src/docbook/tools.ruta.language.syntax.xml uima/ruta/trunk/ruta-ep-ide-ui/src/main/java/org/apache/uima/ruta/ide/validator/LanguageCheckerVisitor.java uima/ruta/trunk/ruta-ep-ide/src/main/antlr3/org/apache/uima/ruta/ide/core/parser/RutaParser.g uima/ruta/trunk/ruta-ep-ide/src/main/java/org/apache/uima/ruta/ide/core/RutaExtensionManager.java uima/ruta/trunk/ruta-ep-ide/src/main/java/org/apache/uima/ruta/ide/core/RutaKeywords.java uima/ruta/trunk/ruta-ep-ide/src/main/java/org/apache/uima/ruta/ide/parser/ast/RutaExpressionList.java uima/ruta/trunk/ruta-ep-ide/src/main/java/org/apache/uima/ruta/ide/parser/ast/RutaStatement.java uima/ruta/trunk/ruta-ep-ide/src/main/java/org/apache/uima/ruta/ide/parser/ast/RutaTypeConstants.java uima/ruta/trunk/ruta-ep-ide/src/main/java/org/apache/uima/ruta/ide/parser/ast/RutaUnaryArithmeticExpression.java uima/ruta/trunk/ruta-ep-ide/src/main/java/org/apache/uima/ruta/ide/parser/ast/StatementFactory.java Modified: uima/ruta/trunk/ruta-core/src/main/antlr3/org/apache/uima/ruta/parser/RutaParser.g URL: http://svn.apache.org/viewvc/uima/ruta/trunk/ruta-core/src/main/antlr3/org/apache/uima/ruta/parser/RutaParser.g?rev=1724465=1724464=1724465=diff == --- uima/ruta/trunk/ruta-core/src/main/antlr3/org/apache/uima/ruta/parser/RutaParser.g (original) +++ uima/ruta/trunk/ruta-core/src/main/antlr3/org/apache/uima/ruta/parser/RutaParser.g Wed Jan 13 17:30:09 2016 @@ -385,11 +385,22 @@ public void setExternalFactory(RutaExter } - - private void addActionMacro(RutaBlock env, String text, Mapdef, List as) { - int i = 0; + + private void addMacroAction(RutaBlock env, String name, Map def, List as) { + env.getEnvironment().addMacroAction(name, def, as); } - + + private boolean isMacroAction(String name, RutaBlock env) { + return env.getEnvironment().isMacroAction(name); + } + + private void addMacroCondition(RutaBlock env, String name, Map def, List cs) { + env.getEnvironment().addMacroCondition(name, def, cs); + } + + private boolean isMacroCondition(String name, RutaBlock env) { + return env.getEnvironment().isMacroCondition(name); + } private Map temporaryVariables = new HashMap<>(); @@ -415,6 +426,8 @@ public void setExternalFactory(RutaExter } } + + } @rulecatch { @@ -479,8 +492,8 @@ statement returns [RutaStatement stmt = : ( stmtDecl = declaration {stmt = stmtDecl;} | stmtVariable = variableDeclaration {stmt = stmtVariable;} - | stmtCM = conditionMacroDeclaration {stmt = stmtCM;} - | stmtAM = actionMacroDeclaration {stmt = stmtAM;} + | stmtCM = macroConditionDeclaration {stmt = stmtCM;} + | stmtAM = macroActionDeclaration {stmt = stmtAM;} | stmtRule = simpleStatement {stmt = stmtRule;} | stmtBlock = blockDeclaration {stmt = stmtBlock;} | stmtExternal = externalBlock {stmt = stmtExternal;} @@ -571,16 +584,23 @@ List vars = new ArrayList def = new LinkedHashMap<>(); +} +: +CONDITION name = Identifier +LPAREN +(argType = varTypeToken argName = Identifier {def.put(argName.getText(),argType.getText());} +(COMMA argType = varTypeToken argName =