OOZIE-2041 Add an admin command to run the PurgeXCommand (abhishekbafna)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/9b07ee82 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/9b07ee82 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/9b07ee82 Branch: refs/heads/oya Commit: 9b07ee82dddaab0ff5562a84b7f5230d11315e9c Parents: 5ed5967 Author: abhisek bafna <aba...@hortonworks.com> Authored: Wed Mar 22 20:17:40 2017 +0530 Committer: abhisek bafna <aba...@hortonworks.com> Committed: Wed Mar 22 20:17:40 2017 +0530 ---------------------------------------------------------------------- .../java/org/apache/oozie/cli/OozieCLI.java | 7 ++ .../org/apache/oozie/cli/ValidationUtil.java | 30 ++++++++ .../org/apache/oozie/client/OozieClient.java | 69 ++++++++++++++++++ .../org/apache/oozie/client/rest/JsonTags.java | 1 + .../apache/oozie/client/rest/RestConstants.java | 6 ++ .../org/apache/oozie/command/PurgeXCommand.java | 6 +- .../org/apache/oozie/service/PurgeService.java | 1 + .../apache/oozie/servlet/BaseAdminServlet.java | 73 +++++++++++++++++--- .../apache/oozie/servlet/V0AdminServlet.java | 16 ++--- .../apache/oozie/servlet/V1AdminServlet.java | 41 ++++------- core/src/main/resources/oozie-default.xml | 8 +++ .../org/apache/oozie/client/TestOozieCLI.java | 55 +++++++++++++++ .../apache/oozie/servlet/TestAdminServlet.java | 32 ++++++++- .../oozie/servlet/TestV1AdminServlet.java | 13 ++++ docs/src/site/twiki/DG_CommandLineTool.twiki | 21 ++++++ docs/src/site/twiki/WebServicesAPI.twiki | 28 ++++++++ release-log.txt | 1 + 17 files changed, 354 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/client/src/main/java/org/apache/oozie/cli/OozieCLI.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java index b0b87ad..38fb84e 100644 --- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java +++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java @@ -151,6 +151,7 @@ public class OozieCLI { public static final String UPDATE_SHARELIB_OPTION = "sharelibupdate"; public static final String LIST_SHARELIB_LIB_OPTION = "shareliblist"; + public static final String PURGE_OPTION = "purge"; public static final String SLA_DISABLE_ALERT = "sladisable"; public static final String SLA_ENABLE_ALERT = "slaenable"; @@ -279,6 +280,8 @@ public class OozieCLI { Option sharelib = new Option(LIST_SHARELIB_LIB_OPTION, false, "List available sharelib that can be specified in a workflow action"); sharelib.setOptionalArg(true); + Option purge = new Option(PURGE_OPTION, true, "purge old oozie workflow, coordinator and bundle records from DB " + + "(parameter unit: day)"); Options adminOptions = new Options(); adminOptions.addOption(oozie); @@ -296,6 +299,7 @@ public class OozieCLI { group.addOption(javaSysProps); group.addOption(metrics); group.addOption(instrumentation); + group.addOption(purge); adminOptions.addOptionGroup(group); addAuthOptions(adminOptions); return adminOptions; @@ -1949,6 +1953,9 @@ public class OozieCLI { } else { printInstrumentation(instrumentation); } + } else if (options.contains(PURGE_OPTION)) { + String purgeOptions = commandLine.getOptionValue(PURGE_OPTION); + System.out.println(wc.purgeCommand(purgeOptions)); } } catch (OozieClientException ex) { http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/client/src/main/java/org/apache/oozie/cli/ValidationUtil.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/cli/ValidationUtil.java b/client/src/main/java/org/apache/oozie/cli/ValidationUtil.java new file mode 100644 index 0000000..b1da2af --- /dev/null +++ b/client/src/main/java/org/apache/oozie/cli/ValidationUtil.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.cli; + +public class ValidationUtil { + + public static int parsePositiveInteger(String iStr) { + int value = Integer.parseInt(iStr); + if (value < 0) { + throw new IllegalArgumentException("Input value should be a positive integer. Value: " + iStr); + } + return value; + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/client/src/main/java/org/apache/oozie/client/OozieClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java index 84f41c9..7370808 100644 --- a/client/src/main/java/org/apache/oozie/client/OozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java @@ -20,6 +20,7 @@ package org.apache.oozie.client; import com.google.common.collect.Lists; import org.apache.oozie.BuildInfo; +import org.apache.oozie.cli.ValidationUtil; import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.client.rest.JsonToBean; import org.apache.oozie.client.rest.RestConstants; @@ -2289,6 +2290,74 @@ public class OozieClient { } + public String purgeCommand(String purgeOptions) throws OozieClientException { + String workflowAge = ""; + String coordAge = ""; + String bundleAge = ""; + String purgeLimit = ""; + String oldCoordAction = ""; + if (purgeOptions != null) { + String options[] = purgeOptions.split(";"); + for (String option : options) { + String pair[] = option.split("="); + if (pair.length < 2) { + throw new OozieClientException(OozieClientException.INVALID_INPUT, + "Invalid purge option pair [" + option + "] specified."); + } + String key = pair[0].toLowerCase(); + String value = pair[1]; + switch (key) { + case "wf": + workflowAge = String.valueOf(ValidationUtil.parsePositiveInteger(value)); + break; + case "coord": + coordAge = String.valueOf(ValidationUtil.parsePositiveInteger(value)); + break; + case "bundle": + bundleAge = String.valueOf(ValidationUtil.parsePositiveInteger(value)); + break; + case "limit": + purgeLimit = String.valueOf(ValidationUtil.parsePositiveInteger(value)); + break; + case "oldcoordaction": + oldCoordAction = value; + break; + default: + throw new OozieClientException(OozieClientException.INVALID_INPUT, + "Invalid purge option [" + key + "] specified."); + } + } + } + PurgeCommand purgeCommand = new PurgeCommand(workflowAge, coordAge, bundleAge, purgeLimit, oldCoordAction); + return purgeCommand.call(); + } + + private class PurgeCommand extends ClientCallable<String> { + + PurgeCommand(String workflowAge, String coordAge, String bundleAge, String purgeLimit, String oldCoordAction) { + super("PUT", RestConstants.ADMIN, RestConstants.ADMIN_PURGE, prepareParams( + RestConstants.PURGE_WF_AGE, workflowAge, + RestConstants.PURGE_COORD_AGE, coordAge, + RestConstants.PURGE_BUNDLE_AGE, bundleAge, + RestConstants.PURGE_LIMIT, purgeLimit, + RestConstants.PURGE_OLD_COORD_ACTION, oldCoordAction + )); + } + + @Override + protected String call(HttpURLConnection conn) throws IOException, OozieClientException { + if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) { + Reader reader = new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8); + JSONObject jsonObject = (JSONObject) JSONValue.parse(reader); + Object msg = jsonObject.get(JsonTags.PURGE); + return msg.toString(); + } else { + handleError(conn); + } + return null; + } + } + /** * Return the Oozie server build version. * http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java b/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java index 8f220b8..ca168e0 100644 --- a/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java +++ b/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java @@ -252,4 +252,5 @@ public interface JsonTags { String COORD_ACTION_FIRST_MISSING_DEPENDENCIES = "blockedOn"; + String PURGE = "purge"; } http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java index 4e38b4a..8ddb1f8 100644 --- a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java +++ b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java @@ -219,4 +219,10 @@ public interface RestConstants { public static final String COORD_ACTION_MISSING_DEPENDENCIES = "missing-dependencies"; + String ADMIN_PURGE = "purge"; + String PURGE_WF_AGE = "wf"; + String PURGE_COORD_AGE = "coord"; + String PURGE_BUNDLE_AGE = "bundle"; + String PURGE_LIMIT = "limit"; + String PURGE_OLD_COORD_ACTION = "oldcoordaction"; } http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/core/src/main/java/org/apache/oozie/command/PurgeXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/PurgeXCommand.java b/core/src/main/java/org/apache/oozie/command/PurgeXCommand.java index ab06fdf..2ba1999 100644 --- a/core/src/main/java/org/apache/oozie/command/PurgeXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/PurgeXCommand.java @@ -142,7 +142,7 @@ public class PurgeXCommand extends XCommand<Void> { @Override protected Void execute() throws CommandException { LOG.info("STARTED Purge to purge Workflow Jobs older than [{0}] days, Coordinator Jobs older than [{1}] days, and Bundle" - + "jobs older than [{2}] days.", wfOlderThan, coordOlderThan, bundleOlderThan); + + " jobs older than [{2}] days.", wfOlderThan, coordOlderThan, bundleOlderThan); // Process parentless workflows to purge them and their children if (!wfList.isEmpty()) { @@ -407,7 +407,7 @@ public class PurgeXCommand extends XCommand<Void> { */ @Override public String getEntityKey() { - return null; + return "purge_command"; } /* (non-Javadoc) @@ -415,7 +415,7 @@ public class PurgeXCommand extends XCommand<Void> { */ @Override protected boolean isLockRequired() { - return false; + return true; } /* (non-Javadoc) http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/core/src/main/java/org/apache/oozie/service/PurgeService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/PurgeService.java b/core/src/main/java/org/apache/oozie/service/PurgeService.java index 6e4a8e8..c2c966b 100644 --- a/core/src/main/java/org/apache/oozie/service/PurgeService.java +++ b/core/src/main/java/org/apache/oozie/service/PurgeService.java @@ -39,6 +39,7 @@ public class PurgeService implements Service { */ public static final String CONF_PURGE_INTERVAL = CONF_PREFIX + "purge.interval"; public static final String PURGE_LIMIT = CONF_PREFIX + "purge.limit"; + public static final String PURGE_COMMAND_ENABLED = CONF_PREFIX + "enable.command.line"; /** * PurgeRunnable is the runnable which is scheduled to run at the configured interval. PurgeCommand is queued to http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java b/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java index 64d3f1f..d263567 100644 --- a/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java @@ -29,20 +29,28 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.oozie.BuildInfo; +import org.apache.oozie.ErrorCode; +import org.apache.oozie.cli.ValidationUtil; import org.apache.oozie.client.rest.JsonBean; import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.command.CommandException; +import org.apache.oozie.command.PurgeXCommand; import org.apache.oozie.service.AuthorizationException; import org.apache.oozie.service.AuthorizationService; +import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.InstrumentationService; import org.apache.oozie.service.JobsConcurrencyService; +import org.apache.oozie.service.PurgeService; import org.apache.oozie.service.Services; import org.apache.oozie.service.ShareLibService; import org.apache.oozie.util.AuthUrlClient; import org.apache.oozie.util.ConfigUtils; import org.apache.oozie.util.Instrumentation; +import org.apache.oozie.util.XLog; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.JSONValue; @@ -50,6 +58,7 @@ import org.json.simple.JSONValue; public abstract class BaseAdminServlet extends JsonRestServlet { private static final long serialVersionUID = 1L; + private static final XLog LOG = XLog.getLog(BaseAdminServlet.class); protected String modeTag; @@ -59,7 +68,13 @@ public abstract class BaseAdminServlet extends JsonRestServlet { } /** - * Change safemode state. + * Oozie admin PUT request for + * change oozie system mode + * admin purge command + * @param request http request + * @param response http response + * @throws ServletException thrown if any servlet error + * @throws IOException thrown if any I/O error */ @Override protected void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { @@ -68,16 +83,19 @@ public abstract class BaseAdminServlet extends JsonRestServlet { request.setAttribute(AUDIT_PARAM, request.getParameter(modeTag)); authorizeRequest(request); - - setOozieMode(request, response, resourceName); - /*if (resourceName.equals(RestConstants.ADMIN_STATUS_RESOURCE)) { - boolean safeMode = Boolean.parseBoolean(request.getParameter(RestConstants.ADMIN_SAFE_MODE_PARAM)); - Services.get().setSafeMode(safeMode); - response.setStatus(HttpServletResponse.SC_OK); + switch (resourceName) { + case RestConstants.ADMIN_STATUS_RESOURCE: + setOozieMode(request, response, resourceName); + break; + case RestConstants.ADMIN_PURGE: + String msg = schedulePurgeCommand(request); + JSONObject jsonObject = new JSONObject(); + jsonObject.put(JsonTags.PURGE, msg); + sendJsonResponse(response, HttpServletResponse.SC_OK, jsonObject); + break; + default: + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0301, resourceName); } - else { - throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0301, resourceName); - }*/ } @@ -165,6 +183,41 @@ public abstract class BaseAdminServlet extends JsonRestServlet { } } + private String schedulePurgeCommand(HttpServletRequest request) throws XServletException { + final String purgeCmdDisabledMsg = "Purge service is not enabled"; + if (!ConfigurationService.getBoolean(PurgeService.PURGE_COMMAND_ENABLED)) { + LOG.warn(purgeCmdDisabledMsg); + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0307, purgeCmdDisabledMsg); + } + String wfAgeStr = request.getParameter(RestConstants.PURGE_WF_AGE); + String coordAgeStr = request.getParameter(RestConstants.PURGE_COORD_AGE); + String bundleAgeStr = request.getParameter(RestConstants.PURGE_BUNDLE_AGE); + String purgeLimitStr = request.getParameter(RestConstants.PURGE_LIMIT); + String oldCoordActionStr = request.getParameter(RestConstants.PURGE_OLD_COORD_ACTION); + + int workflowAge = StringUtils.isBlank(wfAgeStr) ? ConfigurationService.getInt(PurgeService.CONF_OLDER_THAN) + : ValidationUtil.parsePositiveInteger(wfAgeStr); + int coordAge = StringUtils.isBlank(coordAgeStr) ? ConfigurationService.getInt(PurgeService.COORD_CONF_OLDER_THAN) + : ValidationUtil.parsePositiveInteger(coordAgeStr); + int bundleAge = StringUtils.isBlank(bundleAgeStr) ? ConfigurationService.getInt(PurgeService.BUNDLE_CONF_OLDER_THAN) + : ValidationUtil.parsePositiveInteger(bundleAgeStr); + int purgeLimit = StringUtils.isBlank(purgeLimitStr) ? ConfigurationService.getInt(PurgeService.PURGE_LIMIT) + : ValidationUtil.parsePositiveInteger(purgeLimitStr); + boolean purgeOldCoordAction = StringUtils.isBlank(oldCoordActionStr) + ? ConfigurationService.getBoolean(PurgeService.PURGE_OLD_COORD_ACTION) + : Boolean.parseBoolean(oldCoordActionStr); + + LOG.info("Executing oozie purge command."); + PurgeXCommand purgeXCommand = new PurgeXCommand(workflowAge, coordAge, bundleAge, purgeLimit, purgeOldCoordAction); + try { + purgeXCommand.call(); + return "Purge command executed successfully"; + } catch (CommandException e) { + throw new XServletException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getErrorCode(), e.getMessage(), + e.getCause()); + } + } + /** * Gets the list of share lib. * http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/core/src/main/java/org/apache/oozie/servlet/V0AdminServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V0AdminServlet.java b/core/src/main/java/org/apache/oozie/servlet/V0AdminServlet.java index fd573d5..d4a0989 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V0AdminServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V0AdminServlet.java @@ -89,18 +89,10 @@ public class V0AdminServlet extends BaseAdminServlet { @Override protected void setOozieMode(HttpServletRequest request, HttpServletResponse response, String resourceName) throws XServletException { - if (resourceName.equals(RestConstants.ADMIN_STATUS_RESOURCE)) { - boolean safeMode = Boolean.parseBoolean(request.getParameter(modeTag)); - //Services.get().setSafeMode(safeMode); - SYSTEM_MODE sysMode = safeMode == true ? SYSTEM_MODE.NOWEBSERVICE : SYSTEM_MODE.NORMAL; - System.out.println(modeTag + " DDDD " + sysMode); - Services.get().setSystemMode(sysMode); - response.setStatus(HttpServletResponse.SC_OK); - } - else { - throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, - ErrorCode.E0301, resourceName); - } + boolean safeMode = Boolean.parseBoolean(request.getParameter(modeTag)); + SYSTEM_MODE sysMode = safeMode ? SYSTEM_MODE.NOWEBSERVICE : SYSTEM_MODE.NORMAL; + Services.get().setSystemMode(sysMode); + response.setStatus(HttpServletResponse.SC_OK); } /* http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java b/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java index 965a19a..fc3a6bc 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java @@ -18,40 +18,33 @@ package org.apache.oozie.servlet; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.apache.hadoop.conf.Configuration; import org.apache.oozie.ErrorCode; import org.apache.oozie.client.OozieClient.SYSTEM_MODE; -import org.apache.oozie.client.rest.JMSConnectionInfoBean; import org.apache.oozie.client.rest.JsonBean; import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.client.rest.RestConstants; -import org.apache.oozie.jms.JMSConnectionInfo; -import org.apache.oozie.jms.JMSJobEventListener; import org.apache.oozie.service.CallableQueueService; -import org.apache.oozie.service.JMSTopicService; import org.apache.oozie.service.Services; import org.json.simple.JSONArray; import org.json.simple.JSONObject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + public class V1AdminServlet extends BaseAdminServlet { private static final long serialVersionUID = 1L; private static final String INSTRUMENTATION_NAME = "v1admin"; - private static final ResourceInfo RESOURCES_INFO[] = new ResourceInfo[13]; + private static final ResourceInfo RESOURCES_INFO[] = new ResourceInfo[14]; static { RESOURCES_INFO[0] = new ResourceInfo(RestConstants.ADMIN_STATUS_RESOURCE, Arrays.asList("PUT", "GET"), - Arrays.asList(new ParameterInfo(RestConstants.ADMIN_SYSTEM_MODE_PARAM, String.class, true, + Arrays.asList(new ParameterInfo(RestConstants.ADMIN_SYSTEM_MODE_PARAM, String.class, true, Arrays.asList("PUT")))); RESOURCES_INFO[1] = new ResourceInfo(RestConstants.ADMIN_OS_ENV_RESOURCE, Arrays.asList("GET"), Collections.EMPTY_LIST); @@ -77,7 +70,7 @@ public class V1AdminServlet extends BaseAdminServlet { Collections.EMPTY_LIST); RESOURCES_INFO[12] = new ResourceInfo(RestConstants.ADMIN_METRICS_RESOURCE, Arrays.asList("GET"), Collections.EMPTY_LIST); - + RESOURCES_INFO[13] = new ResourceInfo(RestConstants.ADMIN_PURGE, Arrays.asList("PUT"), Collections.EMPTY_LIST); } protected V1AdminServlet(String name) { @@ -114,15 +107,9 @@ public class V1AdminServlet extends BaseAdminServlet { protected void setOozieMode(HttpServletRequest request, HttpServletResponse response, String resourceName) throws XServletException { - if (resourceName.equals(RestConstants.ADMIN_STATUS_RESOURCE)) { - SYSTEM_MODE sysMode = SYSTEM_MODE.valueOf(request.getParameter(modeTag)); - Services.get().setSystemMode(sysMode); - response.setStatus(HttpServletResponse.SC_OK); - } - else { - throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, - ErrorCode.E0301, resourceName); - } + SYSTEM_MODE sysMode = SYSTEM_MODE.valueOf(request.getParameter(modeTag)); + Services.get().setSystemMode(sysMode); + response.setStatus(HttpServletResponse.SC_OK); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 70c34be..03f428a 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -375,6 +375,14 @@ </description> </property> + <property> + <name>oozie.service.PurgeService.enable.command.line</name> + <value>true</value> + <description> + Enable/Disable oozie admin purge command. By default, it is enabled. + </description> + </property> + <!-- RecoveryService --> <property> http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java index d873b96..564db2a 100644 --- a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java +++ b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java @@ -990,6 +990,61 @@ public class TestOozieCLI extends DagServletTestCase { }); } + public void testAdminPurgeCommand() throws Exception { + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + String oozieUrl = getContextURL(); + String[] args = new String[]{"admin", "-purge", "wf=1;coord=2;bundle=3;limit=10;oldCoordAction=true", "-oozie", + oozieUrl}; + String out = runOozieCLIAndGetStdout(args); + assertEquals("Purge command executed successfully" + SYSTEM_LINE_SEPARATOR, out); + return null; + } + }); + + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + String oozieUrl = getContextURL(); + String[] args = new String[]{"admin", "-purge", "wf=1;coord=0;bundle=0;limit=10;oldCoordAction=true", "-oozie", + oozieUrl}; + String out = runOozieCLIAndGetStdout(args); + assertEquals("Purge command executed successfully" + SYSTEM_LINE_SEPARATOR, out); + return null; + } + }); + } + + public void testAdminPurgeCommandNegative() throws Exception { + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + String oozieUrl = getContextURL(); + String[] args = new String[]{"admin", "-purge", "-oozie", oozieUrl}; + String error = runOozieCLIAndGetStderr(args); + assertTrue(error.contains("Missing argument for option: purge")); + + args = new String[]{"admin", "-purge", "invalid=1", "-oozie", oozieUrl}; + error = runOozieCLIAndGetStderr(args); + assertTrue(error.contains("INVALID_INPUT : Invalid purge option [invalid] specified.")); + + args = new String[]{"admin", "-purge", "wf=1;coord=", "-oozie", oozieUrl}; + error = runOozieCLIAndGetStderr(args); + assertTrue(error.contains("INVALID_INPUT : Invalid purge option pair [coord=] specified.")); + + args = new String[]{"admin", "-purge", "wf=1;coord=-1", "-oozie", oozieUrl}; + error = runOozieCLIAndGetStderr(args); + assertTrue(error.contains("Input value should be a positive integer. Value: -1")); + + args = new String[]{"admin", "-purge", "wf=a", "-oozie", oozieUrl}; + error = runOozieCLIAndGetStderr(args); + assertTrue(error.contains("For input string: \"a\"")); + return null; + } + }); + } + public void testClientBuildVersion() throws Exception { String[] args = new String[]{"version"}; String out = runOozieCLIAndGetStdout(args); http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java index ffa3dc1..8e3502c 100644 --- a/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java +++ b/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java @@ -22,8 +22,6 @@ import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.service.Services; import org.apache.oozie.BuildInfo; -import org.apache.oozie.servlet.V0AdminServlet; -import org.apache.oozie.servlet.V0JobServlet; import org.json.simple.JSONObject; import org.json.simple.JSONValue; @@ -267,4 +265,34 @@ public class TestAdminServlet extends DagServletTestCase { } }); } + + public void testPurgeServiceV2() throws Exception { + runTest("/v2/admin/*", V2AdminServlet.class, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + URL url = createURL(RestConstants.ADMIN_PURGE, Collections.EMPTY_MAP); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("PUT"); + assertEquals(HttpServletResponse.SC_OK, connection.getResponseCode()); + return null; + } + }); + } + + public void testPurgeServiceV2Negative() throws Exception { + runTest("/v2/admin/*", V2AdminServlet.class, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + Map<String, String> params = new HashMap<>(); + params.put(RestConstants.PURGE_WF_AGE, "30"); + params.put(RestConstants.PURGE_COORD_AGE, "7"); + params.put(RestConstants.PURGE_BUNDLE_AGE, "-7"); + URL url = createURL(RestConstants.ADMIN_PURGE, params); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("PUT"); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, connection.getResponseCode()); + return null; + } + }); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java index 35568e1..a3f3b9c 100644 --- a/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java +++ b/core/src/test/java/org/apache/oozie/servlet/TestV1AdminServlet.java @@ -259,4 +259,17 @@ public class TestV1AdminServlet extends DagServletTestCase { }); } + + public void testPurgeServiceV1() throws Exception { + runTest("/v1/admin/*", V1AdminServlet.class, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + URL url = createURL(RestConstants.ADMIN_PURGE, Collections.EMPTY_MAP); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("PUT"); + assertEquals(HttpServletResponse.SC_OK, connection.getResponseCode()); + return null; + } + }); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/docs/src/site/twiki/DG_CommandLineTool.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki index 004a35f..6a9e4b4 100644 --- a/docs/src/site/twiki/DG_CommandLineTool.twiki +++ b/docs/src/site/twiki/DG_CommandLineTool.twiki @@ -153,6 +153,8 @@ oozie admin <OPTIONS> : admin operations -systemmode <arg> Supported in Oozie-2.0 or later versions ONLY. Change oozie system mode [NORMAL|NOWEBSERVICE|SAFEMODE] -version show Oozie server build version + -purge <arg> purge old oozie workflow, coordinator and bundle records from DB (parameter unit: day) + wf=<N>\;coord=<N>\;bundle=<N>\;limit=<N>\;oldcoordaction=<true/false> </verbatim> ---+++ Oozie validate command @@ -1513,6 +1515,25 @@ hasn't incremented yet will not show up. *Note:* If Metrics is enabled, then Instrumentation is unavailable. +---+++ Running purge command + +Oozie admin purge command cleans up the Oozie Workflow/Coordinator/Bundle records based on the parameters. +The unit for parameters is day. +Purge command will delete the workflow records (wf=30) older than 30 days, coordinator records (coord=7) older than 7 days and +bundle records (bundle=7) older than 7 days. The limit (limit=10) defines, number of records to be fetch at a time. Turn +(oldCoordAction=true/false) =on/off= coordinator action record purging for long running coordinators. If any of the parameter is +not provided, then it will be taken from the =oozie-default/oozie-site= configuration. + +Example: + +<verbatim> + +$ oozie admin -purge wf=30\;coord=7\;bundle=7\;limit=10\;oldCoordAction=true + +Purge command executed successfully + +</verbatim> + ---++ Validate Operations ---+++ Validating a Workflow XML http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/docs/src/site/twiki/WebServicesAPI.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/WebServicesAPI.twiki b/docs/src/site/twiki/WebServicesAPI.twiki index 03f05d2..a63a68c 100644 --- a/docs/src/site/twiki/WebServicesAPI.twiki +++ b/docs/src/site/twiki/WebServicesAPI.twiki @@ -570,6 +570,34 @@ Content-Type: application/json;charset=UTF-8 ] </verbatim> +---++++ Purge Command + +Oozie admin purge command cleans up the Oozie Workflow/Coordinator/Bundle records based on the parameters. +The unit for parameters is day. + +Purge command will delete the workflow records (wf=30) older than 30 days, coordinator records (coord=7) older than 7 days and +bundle records (bundle=7) older than 7 days. The limit (limit=10) defines, number of records to be fetch at a time. Turn +(oldCoordAction=true/false) =on/off= coordinator action record purging for long running coordinators. If any of the parameter is +not provided, then it will be taken from the =oozie-default/oozie-site= configuration. + +*Request:* + +<verbatim> + +GET /oozie/v2/admin/purge?wf=30&coord=7&bundle=7&limit=10&oldCoordAction=true + +</verbatim> + +*Response:* + +<verbatim> + +{ + "purge": "Purge command executed successfully" +} + +</verbatim> + ---+++ Job and Jobs End-Points _Modified in Oozie v1 WS API_ http://git-wip-us.apache.org/repos/asf/oozie/blob/9b07ee82/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 4a459cb..90b6a02 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2041 Add an admin command to run the PurgeXCommand (abhishekbafna) OOZIE-2393 Allow table drop in hcat prepare (abhishekbafna) OOZIE-2835 TestIOUtils shall not be an XTestCase (asasvari via pbacsko) OOZIE-2817 Increase test case stability in pre-commit job (gezapeti)