svn commit: r1727997 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory: JobFactory.java JobFactoryHelper.java
Author: degenaro
Date: Mon Feb 1 20:24:36 2016
New Revision: 1727997
URL: http://svn.apache.org/viewvc?rev=1727997=rev
Log:
UIMA-4779 DUCC Orchestrator (OR) remove Exceptions and WARNs
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java?rev=1727997=1727996=1727997=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
Mon Feb 1 20:24:36 2016
@@ -20,7 +20,6 @@ package org.apache.uima.ducc.orchestrato
import java.io.File;
import java.util.ArrayList;
-import java.util.Enumeration;
import java.util.List;
import java.util.Map;
@@ -106,21 +105,6 @@ public class JobFactory implements IJobF
return retVal;
}
- private void checkSpec(DuccWorkJob job, JobRequestProperties
jobRequestProperties) {
- String methodName = "checkSpec";
- logger.trace(methodName, job.getDuccId(), "enter");
- jobRequestProperties.normalize();
- Enumeration keys = jobRequestProperties.keys();
- while(keys.hasMoreElements()) {
- String key = (String) keys.nextElement();
- if(!jobRequestProperties.isRecognized(key)) {
- logger.warn(methodName, job.getDuccId(),
"unrecognized: "+key);
- }
- }
- logger.trace(methodName, job.getDuccId(), "exit");
- return;
- }
-
private ArrayList toArrayList(String overrides) {
String methodName = "toArrayList";
logger.trace(methodName, null, "enter");
@@ -502,7 +486,7 @@ public class JobFactory implements IJobF
private DuccWorkJob create(CommonConfiguration common,
JobRequestProperties jobRequestProperties, DuccWorkJob job) {
String methodName = "create";
- checkSpec(job, jobRequestProperties);
+ jobRequestProperties.normalize();
DuccType duccType = job.getDuccType();
// Service Deployment Type
if(jobRequestProperties.containsKey(ServiceRequestProperties.key_service_type_custom))
{
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java?rev=1727997=1727996=1727997=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java
Mon Feb 1 20:24:36 2016
@@ -44,11 +44,22 @@ public class JobFactoryHelper {
public static long getByteSizeJobProcess(String process_memory_size) {
String location = "getByteSizeJobProcess";
long retVal = getDefaultByteSizeJobProcess();
- try {
- retVal = Long.parseLong(process_memory_size);
+ if(process_memory_size != null) {
+ String memory_size = process_memory_size.trim();
+ if(memory_size.length() > 0) {
+ try {
+ retVal =
Long.parseLong(process_memory_size);
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ }
+ }
+ else {
+ logger.info(location, jobid, "memory size:
empty");
+ }
}
- catch(Exception e) {
- logger.error(location, jobid, e);
+ else {
+ logger.info(location, jobid, "memory size: null");
}
logger.debug(location, jobid, retVal);
return retVal;
svn commit: r1727979 [2/3] - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook: ./ images/ducc-internals/ part5/
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex?rev=1727979=auto
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex
(added)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex
Mon Feb 1 17:36:08 2016
@@ -0,0 +1,1109 @@
+%
+% Licensed to the Apache Software Foundation (ASF) under one
+% or more contributor license agreements. See the NOTICE file
+% distributed with this work for additional information
+% regarding copyright ownership. The ASF licenses this file
+% to you under the Apache License, Version 2.0 (the
+% "License"); you may not use this file except in compliance
+% with the License. You may obtain a copy of the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing,
+% software distributed under the License is distributed on an
+% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+% KIND, either express or implied. See the License for the
+% specific language governing permissions and limitations
+% under the License.
+%
+
+
+This chapter provides architectural and implementation details for the DUCC
+Resource Manager, referred to as the ``RM''.
+\section{Introduction}
+
+The DUCC Resource Manager is responsible for apportioning cluster resources to
the
+collection of ``work'' to be run by users. Work is classified into several
categories. As
+exposed in the public interface, these categories are:
+
+\begin{description}
+ \item[Fair-Share Job] This is a UIMA/AS job, consisting of a minimum of two
processes and
+a potential maximum of as many processes as physically fit on a cluster.
The work
+executed by the processes is parallel, enabling the RM to expand or
contract
+the job by allocating or deallocating processes as needed to balance the
load.
+
+Load is balanced using a weighted fair-share policy in which all users are
apportioned an
+equitable amount of the cluster resources, where ``cluster resources''
refers only to real
+memory on the cluster nodes.
+
+ \item[Service Instance] This is any arbitrary process that DUCC manages as a
``service''.
+Services are registered with the Service Manager and may be comprised of
multiple physical
+processes. (See the DuccBook for details of DUCC Service Management.)
The RM schedules these
+processes as singletons, using a non-preemptive policy (FIXED\_SHARE or
RESERVE).
+
+ \item[Arbitrary Process or ``Managed Reservation''] These are singleton
processes of any type, scheduled
+using a FIXED\_SHARE policy.
+
+ \item[Fixed-Share Job] This is a UIMA-AS job scheduled with a
non-preemptable, i.e. FIXED\_SHARE
+policy.
+
+ \item[Reservation] This is a request for a dedicated full machine.
+\end{description}
+
+The RM is a memory scheduler only. The use case which justifies DUCC is
UIMA-AS jobs, each of
+which consists a variable number of parallel processes, each of which requires
large amounts of memory, usually
+on the order of 16GB or more. Memory requirements completely overwhelm other
resource
+requirements, so that jobs scheduled by their declared memory sizes usually
get sufficient
+other resource such as CPU.
+
+\section{Vocabulary}
+In order to understand RM it is necessary to understand some of the
language used in RM.
+
+\begin{description}
+\item[quantum] This is the smallest memory size of an allocation, defined
in multiples of GB. It
+ is defined globally in {\em ducc.properties} and may be overridden in
{\em ducc.classes} for
+ top-level nodepools. See the DuccBook for more details.
+
+ Note that although DUCC defines a quantum, most of the RM does not
+ use quantum at all; instead it generalizes quantum into {\em qshare},
{\em nshare},
+ and {\em order} as defined below. When a schedule is returned to the
Orchestrator, the
+ allocations, in terms of quanta, are translated back to memory
allocations using this
+ configured {\em quantum}.
+
+\item[qshare] This is an abstract memory allocation representing exactly
one {\em quantum}. Memory
+ allocations are made in terms of some multiple of {\em qshares}.
+
+\item[nshare] This is an allocation which consists of one or more
co-located {\em qshares}. When
+ exposed outside of RM this is usually thought of as a ``process''. It
means, literally,
+ ``n qshares''.
+
+Be careful, an {\em nshare} is NOT a process, it is an allocation that
can be put to
+any use or to no use if desired. The RM does not care what an {\em
nshare} is
svn commit: r1727979 [3/3] - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook: ./ images/ducc-internals/ part5/
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex?rev=1727979=auto
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex
(added)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex
Mon Feb 1 17:36:08 2016
@@ -0,0 +1,578 @@
+%
+% Licensed to the Apache Software Foundation (ASF) under one
+% or more contributor license agreements. See the NOTICE file
+% distributed with this work for additional information
+% regarding copyright ownership. The ASF licenses this file
+% to you under the Apache License, Version 2.0 (the
+% "License"); you may not use this file except in compliance
+% with the License. You may obtain a copy of the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing,
+% software distributed under the License is distributed on an
+% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+% KIND, either express or implied. See the License for the
+% specific language governing permissions and limitations
+% under the License.
+%
+
+% \section{DUCC Service Manager}
+This section describes the architecture and internal structure of the
+DUCC Service Manager, referred to as the ``SM''.
+
+\section{Introduction}
+The SM function is to insure that any services needed by
+DUCC jobs are running and functional at the time they are needed by
+jobs. Previous to the incarnation of the SM it was necessary for
+users to manually invoke the processes implementing their services. If
+these processes were to crash, jobs dependent on them would stop until
+some human was able to restart the service. If the operating system,
+or batch system supporting the jobs (DUCC, in our case) was to
+be restarted, users would again have to manually start the services.
+
+By ``registering'' a service with the SM, a user can trust DUCC to
+keep the service alive and functional across all manner of faults and
+system restarts. As well, the SM has a mechanism for ``testing'' a
+service to determine if it is operational, and to inform the DUCC
+Web Server when it is not.
+
+If a user submits a job that declares a dependency on a service, the SM
+is able to start the service as needed, and is able to stop the service
+when no longer needed, freeing resources.
+
+In essence, the SM can be thought of as a ``proxy user'' dedicated
+to insuring that services are always available when needed.
+
+\section{Architectural Overview}
+
+Figure ~\ref{fig:sm-structure} below shows the high-level object,
+threading, and process structure of SM and should be referenced
+while reading this document.
+
+The SM can be pictured as being composed of four major parts:
+\begin{enumerate}
+ \item Initialization and interaction with external components.
+External components include user requests and other DUCC components
such as
+the Orchestrator.
+ \item A ``Service Instance'' manager. This part resolves
+dependencies on services, starts and stops service instances
+according to the needs of jobs and the policies declared in
+the service registries, and handles the service instance
+lifetimes.
+ \item A ``Service Health'' manager. This part continually
+``tests'' services to determine whether they are
+functional. This is referred to as the ``pinger'' and the
+test is known as a ``ping''.
+ \item A ``CLI Handler'' which reacts to requests from users.
+\end{enumerate}
+
+\begin{figure}[H]
+ \centering
+ \includegraphics[width=5.5in]{images/ducc-internals/sm-structure.png}
+ \caption{Service Manager Structure}
+ \label{fig:sm-structure}
+\end{figure}
+
+ The terminology around Services can be confusing. We review the ideas here.
+
+ There are three ``countable'' entities involved in services.
+ \begin{description}
+ \item[Service Registration] When a service is ``registered'' the Service
Manager assigns
+ a new, unique {\em Registration ID} to the registration. This ID is
associated with, and
+ remains with, the service throughout its lifetime and beyond when it is
archived.
+
+ \item[Service Instance] When the Service Manager starts a service it
issues a series of
+ ``submit'' orders to the Orchestrator, one for each {\em Service
Instance}. All these
+ instances are associated with the {\em Service Registration}. The
orchestrator assigns
+ a unique ID to each service instance, which is also
svn commit: r1727979 [1/3] - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook: ./ images/ducc-internals/ part5/
Author: challngr Date: Mon Feb 1 17:36:08 2016 New Revision: 1727979 URL: http://svn.apache.org/viewvc?rev=1727979=rev Log: UIMA-4777 Internals documentation. Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/ uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/sm-structure.png (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/internals-book.tex uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-database.tex uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png?rev=1727979=auto == Binary file - no diff available. Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png -- svn:mime-type = application/octet-stream Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png?rev=1727979=auto == Binary file - no diff available. Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png -- svn:mime-type = application/octet-stream Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png?rev=1727979=auto == Binary file - no diff available. Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png -- svn:mime-type = application/octet-stream Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png?rev=1727979=auto == Binary file - no diff available. Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png -- svn:mime-type = application/octet-stream Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd?rev=1727979=auto == Binary file - no diff available. Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd -- svn:mime-type = application/octet-stream Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/sm-structure.png URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/sm-structure.png?rev=1727979=auto
svn commit: r1727969 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws: ./ server/ server/nodeviz/ types/
Author: degenaro
Date: Mon Feb 1 16:32:01 2016
New Revision: 1727969
URL: http://svn.apache.org/viewvc?rev=1727969=rev
Log:
UIMA-4776 DUCC Web Server (WS) should key machines by IP address not host name
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/Ip.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java?rev=1727969=1727968=1727969=diff
==
---
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
Mon Feb 1 16:32:01 2016
@@ -18,7 +18,6 @@
*/
package org.apache.uima.ducc.ws;
-import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -52,21 +51,19 @@ public class DuccMachinesData {
private static DuccLogger logger =
DuccLoggerComponents.getWsLogger(DuccMachinesData.class.getName());
private static DuccId jobid = null;
- private static ConcurrentSkipListMap sortedMachines
= new ConcurrentSkipListMap();
- private static ConcurrentSkipListMap
unsortedMachines = new ConcurrentSkipListMap();
- private static ConcurrentSkipListMap
summaryMachines = new ConcurrentSkipListMap();
+ private static ConcurrentSkipListMap sortedMachines =
new ConcurrentSkipListMap();
+ private static ConcurrentSkipListMap unsortedMachines =
new ConcurrentSkipListMap();
+ private static ConcurrentSkipListMap
summaryMachines = new ConcurrentSkipListMap();
private static AtomicLong memTotal = new AtomicLong(0);
private static AtomicLong memFree = new AtomicLong(0);
private static AtomicLong swapInuse = new AtomicLong(0);
private static AtomicLong swapFree = new AtomicLong(0);
-
- private String domain = "";
private static DuccMachinesData duccMachinesData = new
DuccMachinesData();
- private static ConcurrentSkipListMap ipToNameMap = new
ConcurrentSkipListMap();
- private static ConcurrentSkipListMap nameToIpMap = new
ConcurrentSkipListMap();
+ private static ConcurrentSkipListMap ipToNameMap = new
ConcurrentSkipListMap();
+ private static ConcurrentSkipListMap nameToIpMap = new
ConcurrentSkipListMap();
private static ConcurrentSkipListMap isSwapping = new
ConcurrentSkipListMap();
private static
ConcurrentSkipListMap>
ipToNodeUsersInfoMap = new
ConcurrentSkipListMap>();
@@ -79,12 +76,12 @@ public class DuccMachinesData {
return isSwapping.containsKey(ip);
}
- public ConcurrentSkipListMap getMachines() {
+ public ConcurrentSkipListMap getMachines() {
return unsortedMachines;
}
- public ConcurrentSkipListMap getSortedMachines() {
- ConcurrentSkipListMap retVal =
sortedMachines;
+ public ConcurrentSkipListMap getSortedMachines() {
+ ConcurrentSkipListMap retVal = sortedMachines;
return retVal;
}
@@ -92,9 +89,9 @@ public class DuccMachinesData {
String location = "updateSortedMachines";
logger.debug(location, jobid, "start");
try {
- ConcurrentSkipListMap map = new
ConcurrentSkipListMap();
-
