svn commit: r1727997 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory: JobFactory.java JobFactoryHelper.java
Author: degenaro Date: Mon Feb 1 20:24:36 2016 New Revision: 1727997 URL: http://svn.apache.org/viewvc?rev=1727997&view=rev Log: UIMA-4779 DUCC Orchestrator (OR) remove Exceptions and WARNs Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java?rev=1727997&r1=1727996&r2=1727997&view=diff == --- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java Mon Feb 1 20:24:36 2016 @@ -20,7 +20,6 @@ package org.apache.uima.ducc.orchestrato import java.io.File; import java.util.ArrayList; -import java.util.Enumeration; import java.util.List; import java.util.Map; @@ -106,21 +105,6 @@ public class JobFactory implements IJobF return retVal; } - private void checkSpec(DuccWorkJob job, JobRequestProperties jobRequestProperties) { - String methodName = "checkSpec"; - logger.trace(methodName, job.getDuccId(), "enter"); - jobRequestProperties.normalize(); - Enumeration keys = jobRequestProperties.keys(); - while(keys.hasMoreElements()) { - String key = (String) keys.nextElement(); - if(!jobRequestProperties.isRecognized(key)) { - logger.warn(methodName, job.getDuccId(), "unrecognized: "+key); - } - } - logger.trace(methodName, job.getDuccId(), "exit"); - return; - } - private ArrayList toArrayList(String overrides) { String methodName = "toArrayList"; logger.trace(methodName, null, "enter"); @@ -502,7 +486,7 @@ public class JobFactory implements IJobF private DuccWorkJob create(CommonConfiguration common, JobRequestProperties jobRequestProperties, DuccWorkJob job) { String methodName = "create"; - checkSpec(job, jobRequestProperties); + jobRequestProperties.normalize(); DuccType duccType = job.getDuccType(); // Service Deployment Type if(jobRequestProperties.containsKey(ServiceRequestProperties.key_service_type_custom)) { Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java?rev=1727997&r1=1727996&r2=1727997&view=diff == --- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactoryHelper.java Mon Feb 1 20:24:36 2016 @@ -44,11 +44,22 @@ public class JobFactoryHelper { public static long getByteSizeJobProcess(String process_memory_size) { String location = "getByteSizeJobProcess"; long retVal = getDefaultByteSizeJobProcess(); - try { - retVal = Long.parseLong(process_memory_size); + if(process_memory_size != null) { + String memory_size = process_memory_size.trim(); + if(memory_size.length() > 0) { + try { + retVal = Long.parseLong(process_memory_size); + } + catch(Exception e) { + logger.error(location, jobid, e); + } + } + else { + logger.info(location, jobid, "memory size: empty"); + } } - catch(Exception e) { - logger.error(location, jobid, e); + else { + logger.info(location, jobid, "memory size: null"); } logger.debug(location, jobid, retVal); ret
svn commit: r1727979 [2/3] - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook: ./ images/ducc-internals/ part5/
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex?rev=1727979&view=auto == --- uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex (added) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex Mon Feb 1 17:36:08 2016 @@ -0,0 +1,1109 @@ +% +% Licensed to the Apache Software Foundation (ASF) under one +% or more contributor license agreements. See the NOTICE file +% distributed with this work for additional information +% regarding copyright ownership. The ASF licenses this file +% to you under the Apache License, Version 2.0 (the +% "License"); you may not use this file except in compliance +% with the License. You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, +% software distributed under the License is distributed on an +% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +% KIND, either express or implied. See the License for the +% specific language governing permissions and limitations +% under the License. +% + + +This chapter provides architectural and implementation details for the DUCC +Resource Manager, referred to as the ``RM''. +\section{Introduction} + +The DUCC Resource Manager is responsible for apportioning cluster resources to the +collection of ``work'' to be run by users. Work is classified into several categories. As +exposed in the public interface, these categories are: + +\begin{description} + \item[Fair-Share Job] This is a UIMA/AS job, consisting of a minimum of two processes and +a potential maximum of as many processes as physically fit on a cluster. The work +executed by the processes is parallel, enabling the RM to expand or contract +the job by allocating or deallocating processes as needed to balance the load. + +Load is balanced using a weighted fair-share policy in which all users are apportioned an +equitable amount of the cluster resources, where ``cluster resources'' refers only to real +memory on the cluster nodes. + + \item[Service Instance] This is any arbitrary process that DUCC manages as a ``service''. +Services are registered with the Service Manager and may be comprised of multiple physical +processes. (See the DuccBook for details of DUCC Service Management.) The RM schedules these +processes as singletons, using a non-preemptive policy (FIXED\_SHARE or RESERVE). + + \item[Arbitrary Process or ``Managed Reservation''] These are singleton processes of any type, scheduled +using a FIXED\_SHARE policy. + + \item[Fixed-Share Job] This is a UIMA-AS job scheduled with a non-preemptable, i.e. FIXED\_SHARE +policy. + + \item[Reservation] This is a request for a dedicated full machine. +\end{description} + +The RM is a memory scheduler only. The use case which justifies DUCC is UIMA-AS jobs, each of +which consists a variable number of parallel processes, each of which requires large amounts of memory, usually +on the order of 16GB or more. Memory requirements completely overwhelm other resource +requirements, so that jobs scheduled by their declared memory sizes usually get sufficient +other resource such as CPU. + +\section{Vocabulary} +In order to understand RM it is necessary to understand some of the language used in RM. + +\begin{description} +\item[quantum] This is the smallest memory size of an allocation, defined in multiples of GB. It + is defined globally in {\em ducc.properties} and may be overridden in {\em ducc.classes} for + top-level nodepools. See the DuccBook for more details. + + Note that although DUCC defines a quantum, most of the RM does not + use quantum at all; instead it generalizes quantum into {\em qshare}, {\em nshare}, + and {\em order} as defined below. When a schedule is returned to the Orchestrator, the + allocations, in terms of quanta, are translated back to memory allocations using this + configured {\em quantum}. + +\item[qshare] This is an abstract memory allocation representing exactly one {\em quantum}. Memory + allocations are made in terms of some multiple of {\em qshares}. + +\item[nshare] This is an allocation which consists of one or more co-located {\em qshares}. When + exposed outside of RM this is usually thought of as a ``process''. It means, literally, + ``n qshares''. + +Be careful, an {\em nshare} is NOT a process, it is an allocation that can be put to +any use or to no use if desired. The RM does not care what an {\em nshare} is
svn commit: r1727979 [3/3] - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook: ./ images/ducc-internals/ part5/
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex?rev=1727979&view=auto == --- uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex (added) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex Mon Feb 1 17:36:08 2016 @@ -0,0 +1,578 @@ +% +% Licensed to the Apache Software Foundation (ASF) under one +% or more contributor license agreements. See the NOTICE file +% distributed with this work for additional information +% regarding copyright ownership. The ASF licenses this file +% to you under the Apache License, Version 2.0 (the +% "License"); you may not use this file except in compliance +% with the License. You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, +% software distributed under the License is distributed on an +% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +% KIND, either express or implied. See the License for the +% specific language governing permissions and limitations +% under the License. +% + +% \section{DUCC Service Manager} +This section describes the architecture and internal structure of the +DUCC Service Manager, referred to as the ``SM''. + +\section{Introduction} +The SM function is to insure that any services needed by +DUCC jobs are running and functional at the time they are needed by +jobs. Previous to the incarnation of the SM it was necessary for +users to manually invoke the processes implementing their services. If +these processes were to crash, jobs dependent on them would stop until +some human was able to restart the service. If the operating system, +or batch system supporting the jobs (DUCC, in our case) was to +be restarted, users would again have to manually start the services. + +By ``registering'' a service with the SM, a user can trust DUCC to +keep the service alive and functional across all manner of faults and +system restarts. As well, the SM has a mechanism for ``testing'' a +service to determine if it is operational, and to inform the DUCC +Web Server when it is not. + +If a user submits a job that declares a dependency on a service, the SM +is able to start the service as needed, and is able to stop the service +when no longer needed, freeing resources. + +In essence, the SM can be thought of as a ``proxy user'' dedicated +to insuring that services are always available when needed. + +\section{Architectural Overview} + +Figure ~\ref{fig:sm-structure} below shows the high-level object, +threading, and process structure of SM and should be referenced +while reading this document. + +The SM can be pictured as being composed of four major parts: +\begin{enumerate} + \item Initialization and interaction with external components. +External components include user requests and other DUCC components such as +the Orchestrator. + \item A ``Service Instance'' manager. This part resolves +dependencies on services, starts and stops service instances +according to the needs of jobs and the policies declared in +the service registries, and handles the service instance +lifetimes. + \item A ``Service Health'' manager. This part continually +``tests'' services to determine whether they are +functional. This is referred to as the ``pinger'' and the +test is known as a ``ping''. + \item A ``CLI Handler'' which reacts to requests from users. +\end{enumerate} + +\begin{figure}[H] + \centering + \includegraphics[width=5.5in]{images/ducc-internals/sm-structure.png} + \caption{Service Manager Structure} + \label{fig:sm-structure} +\end{figure} + + The terminology around Services can be confusing. We review the ideas here. + + There are three ``countable'' entities involved in services. + \begin{description} + \item[Service Registration] When a service is ``registered'' the Service Manager assigns + a new, unique {\em Registration ID} to the registration. This ID is associated with, and + remains with, the service throughout its lifetime and beyond when it is archived. + + \item[Service Instance] When the Service Manager starts a service it issues a series of + ``submit'' orders to the Orchestrator, one for each {\em Service Instance}. All these + instances are associated with the {\em Service Registration}. The orchestrator assigns + a unique ID to each service instance, which is also perma
svn commit: r1727979 [1/3] - in /uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook: ./ images/ducc-internals/ part5/
Author: challngr Date: Mon Feb 1 17:36:08 2016 New Revision: 1727979 URL: http://svn.apache.org/viewvc?rev=1727979&view=rev Log: UIMA-4777 Internals documentation. Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/ uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/sm-structure.png (with props) uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/internals-book.tex uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-database.tex uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-rm.tex uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part5/ducc-pops-component-sm.tex Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png?rev=1727979&view=auto == Binary file - no diff available. Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/db-structure.png -- svn:mime-type = application/octet-stream Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png?rev=1727979&view=auto == Binary file - no diff available. Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-1.png -- svn:mime-type = application/octet-stream Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png?rev=1727979&view=auto == Binary file - no diff available. Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure-2.png -- svn:mime-type = application/octet-stream Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png?rev=1727979&view=auto == Binary file - no diff available. Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.png -- svn:mime-type = application/octet-stream Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd?rev=1727979&view=auto == Binary file - no diff available. Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/rm-structure.vsd -- svn:mime-type = application/octet-stream Added: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/sm-structure.png URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/images/ducc-internals/sm-structure.png?rev=1727979&view=auto
svn commit: r1727969 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws: ./ server/ server/nodeviz/ types/
Author: degenaro Date: Mon Feb 1 16:32:01 2016 New Revision: 1727969 URL: http://svn.apache.org/viewvc?rev=1727969&view=rev Log: UIMA-4776 DUCC Web Server (WS) should key machines by IP address not host name Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/Ip.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java?rev=1727969&r1=1727968&r2=1727969&view=diff == --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java Mon Feb 1 16:32:01 2016 @@ -18,7 +18,6 @@ */ package org.apache.uima.ducc.ws; -import java.net.InetAddress; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -52,21 +51,19 @@ public class DuccMachinesData { private static DuccLogger logger = DuccLoggerComponents.getWsLogger(DuccMachinesData.class.getName()); private static DuccId jobid = null; - private static ConcurrentSkipListMap sortedMachines = new ConcurrentSkipListMap(); - private static ConcurrentSkipListMap unsortedMachines = new ConcurrentSkipListMap(); - private static ConcurrentSkipListMap summaryMachines = new ConcurrentSkipListMap(); + private static ConcurrentSkipListMap sortedMachines = new ConcurrentSkipListMap(); + private static ConcurrentSkipListMap unsortedMachines = new ConcurrentSkipListMap(); + private static ConcurrentSkipListMap summaryMachines = new ConcurrentSkipListMap(); private static AtomicLong memTotal = new AtomicLong(0); private static AtomicLong memFree = new AtomicLong(0); private static AtomicLong swapInuse = new AtomicLong(0); private static AtomicLong swapFree = new AtomicLong(0); - - private String domain = ""; private static DuccMachinesData duccMachinesData = new DuccMachinesData(); - private static ConcurrentSkipListMap ipToNameMap = new ConcurrentSkipListMap(); - private static ConcurrentSkipListMap nameToIpMap = new ConcurrentSkipListMap(); + private static ConcurrentSkipListMap ipToNameMap = new ConcurrentSkipListMap(); + private static ConcurrentSkipListMap nameToIpMap = new ConcurrentSkipListMap(); private static ConcurrentSkipListMap isSwapping = new ConcurrentSkipListMap(); private static ConcurrentSkipListMap> ipToNodeUsersInfoMap = new ConcurrentSkipListMap>(); @@ -79,12 +76,12 @@ public class DuccMachinesData { return isSwapping.containsKey(ip); } - public ConcurrentSkipListMap getMachines() { + public ConcurrentSkipListMap getMachines() { return unsortedMachines; } - public ConcurrentSkipListMap getSortedMachines() { - ConcurrentSkipListMap retVal = sortedMachines; + public ConcurrentSkipListMap getSortedMachines() { + ConcurrentSkipListMap retVal = sortedMachines; return retVal; } @@ -92,9 +89,9 @@ public class DuccMachinesData { String location = "updateSortedMachines"; logger.debug(location, jobid, "start"); try { - ConcurrentSkipListMap map = new ConcurrentSkipListMap(); - for(Entry entry : unsortedMachines.entrySet()) { - String value = entry.getKey(); + ConcurrentSkipListMap map = new ConcurrentSkipListMap(); + for(Entry entry : unsortedMachines.entrySet()) { + Ip value = entry.getKey(); MachineInfo key = entry.getValue(); map.put(key, value); logger.debug(location, jobid, "put: "+value); @@ -127,24 +124,14 @@ public class DuccMachinesData {