Merge branch 'apache_forked_master' into apache_forked_oya Conflicts: core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java core/src/main/java/org/apache/oozie/service/ShareLibService.java core/src/main/java/org/apache/oozie/util/JobUtils.java core/src/main/java/org/apache/oozie/util/NamedThreadFactory.java core/src/main/resources/oozie-default.xml core/src/test/java/org/apache/oozie/QueryServlet.java core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java docs/src/site/twiki/DG_QuickStart.twiki docs/src/site/twiki/ENG_Building.twiki hadooplibs/hadoop-auth-1/pom.xml hadooplibs/hadoop-auth-2/pom.xml hadooplibs/hadoop-distcp-1/pom.xml hadooplibs/hadoop-distcp-2/pom.xml hadooplibs/hadoop-distcp-3/pom.xml hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java hadooplibs/hadoop-utils-1/pom.xml hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java hadooplibs/hadoop-utils-2/pom.xml hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java hadooplibs/hadoop-utils-3/pom.xml hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java hadooplibs/pom.xml pom.xml release-log.txt sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputFormat.java sharelib/spark/pom.xml sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
Change-Id: I6697e098f84051365e3b11d93eb6effe124ed47d Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/c49f382b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/c49f382b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/c49f382b Branch: refs/heads/oya Commit: c49f382bef2d0b101dda9a40cdf36c4cba454f85 Parents: ed72570 3eca3c2 Author: Peter Bacsko <pbac...@cloudera.com> Authored: Fri Sep 30 12:33:27 2016 +0200 Committer: Peter Bacsko <pbac...@cloudera.com> Committed: Fri Sep 30 12:33:27 2016 +0200 ---------------------------------------------------------------------- bin/test-patch | 2 +- bin/test-patch-10-compile | 4 +- client/pom.xml | 4 +- .../java/org/apache/oozie/cli/OozieCLI.java | 5 +- .../java/org/apache/oozie/client/BundleJob.java | 4 +- .../apache/oozie/client/CoordinatorAction.java | 2 +- .../org/apache/oozie/client/CoordinatorJob.java | 4 +- .../main/java/org/apache/oozie/client/Job.java | 6 +- .../org/apache/oozie/client/OozieClient.java | 23 +- .../java/org/apache/oozie/client/SLAEvent.java | 46 +- .../org/apache/oozie/client/WorkflowAction.java | 2 +- .../org/apache/oozie/client/WorkflowJob.java | 2 +- .../org/apache/oozie/client/XOozieClient.java | 31 +- .../org/apache/oozie/client/rest/JsonTags.java | 438 +++--- .../org/apache/oozie/client/rest/JsonUtils.java | 9 +- .../apache/oozie/client/rest/RestConstants.java | 192 +-- .../src/main/resources/oozie-workflow-0.5.xsd | 1 + core/pom.xml | 45 +- .../org/apache/oozie/CoordinatorEngine.java | 6 +- .../main/java/org/apache/oozie/DagEngine.java | 1 + .../main/java/org/apache/oozie/ErrorCode.java | 4 +- .../org/apache/oozie/LocalOozieClientCoord.java | 25 + .../org/apache/oozie/action/ActionExecutor.java | 46 +- .../action/hadoop/DistcpActionExecutor.java | 24 +- .../oozie/action/hadoop/FsActionExecutor.java | 9 +- .../action/hadoop/HCatCredentialHelper.java | 8 +- .../oozie/action/hadoop/HCatCredentials.java | 49 +- .../oozie/action/hadoop/HadoopELFunctions.java | 5 - .../action/hadoop/Hive2ActionExecutor.java | 30 +- .../oozie/action/hadoop/Hive2Credentials.java | 17 +- .../oozie/action/hadoop/HiveActionExecutor.java | 28 +- .../oozie/action/hadoop/JavaActionExecutor.java | 75 +- .../action/hadoop/LauncherMapperHelper.java | 21 +- .../action/hadoop/MapReduceActionExecutor.java | 6 + .../action/hadoop/ShellActionExecutor.java | 10 + .../action/hadoop/SparkActionExecutor.java | 30 +- .../action/hadoop/SqoopActionExecutor.java | 25 +- .../action/oozie/SubWorkflowActionExecutor.java | 13 +- .../org/apache/oozie/client/rest/JsonBean.java | 4 +- .../bundle/BundleCoordSubmitXCommand.java | 77 + .../command/bundle/BundleStartXCommand.java | 3 +- .../command/coord/CoordActionCheckXCommand.java | 8 +- .../coord/CoordActionInputCheckXCommand.java | 13 +- .../coord/CoordActionsIgnoreXCommand.java | 2 +- .../oozie/command/coord/CoordRerunXCommand.java | 3 +- .../command/coord/CoordSubmitXCommand.java | 6 +- .../oozie/command/wf/ActionCheckXCommand.java | 2 +- .../oozie/command/wf/ActionEndXCommand.java | 2 +- .../oozie/command/wf/ActionStartXCommand.java | 40 +- .../apache/oozie/command/wf/ActionXCommand.java | 92 +- .../command/wf/CompletedActionXCommand.java | 2 +- .../command/wf/ForkedActionStartXCommand.java | 7 + .../apache/oozie/command/wf/SignalXCommand.java | 65 +- .../oozie/command/wf/SubmitMRXCommand.java | 3 - .../apache/oozie/command/wf/SubmitXCommand.java | 24 + .../wf/WorkflowNotificationXCommand.java | 6 + .../oozie/compression/CompressionCodec.java | 8 +- .../input/dependency/CoordInputDependency.java | 34 +- .../input/logic/CoordInputLogicEvaluator.java | 6 +- .../org/apache/oozie/dependency/URIHandler.java | 34 +- .../dependency/hcat/HCatDependencyCache.java | 20 +- .../java/org/apache/oozie/event/EventQueue.java | 18 +- .../oozie/executor/jpa/BulkJPAExecutor.java | 91 +- .../jpa/BundleJobInfoGetJPAExecutor.java | 14 +- .../executor/jpa/BundleJobQueryExecutor.java | 2 +- .../jpa/CoordJobInfoGetJPAExecutor.java | 14 +- .../apache/oozie/executor/jpa/JPAExecutor.java | 4 +- .../jpa/WorkflowsJobGetJPAExecutor.java | 42 +- .../org/apache/oozie/jms/ConnectionContext.java | 18 +- .../org/apache/oozie/jms/MessageHandler.java | 2 +- .../java/org/apache/oozie/lock/MemoryLocks.java | 82 +- .../oozie/service/ActionConfFileType.java | 56 + .../org/apache/oozie/service/ActionService.java | 5 + .../oozie/service/AuthorizationService.java | 6 +- .../oozie/service/CallableQueueService.java | 8 +- .../oozie/service/ConfigurationService.java | 1 + .../oozie/service/HadoopAccessorService.java | 98 +- .../oozie/service/LiteWorkflowStoreService.java | 13 +- .../oozie/service/MemoryLocksService.java | 9 +- .../apache/oozie/service/RecoveryService.java | 107 +- .../apache/oozie/service/SchedulerService.java | 11 +- .../oozie/service/SchemaCheckerService.java | 8 +- .../java/org/apache/oozie/service/Service.java | 12 +- .../apache/oozie/service/ShareLibService.java | 8 +- .../apache/oozie/service/ZKLocksService.java | 88 +- .../org/apache/oozie/service/ZKUUIDService.java | 2 +- .../org/apache/oozie/sla/SLACalculator.java | 2 +- .../apache/oozie/sla/SLACalculatorMemory.java | 13 +- .../org/apache/oozie/sla/SLASummaryBean.java | 8 +- .../apache/oozie/store/StoreStatusFilter.java | 138 +- .../org/apache/oozie/util/Instrumentable.java | 2 +- .../apache/oozie/util/JaasConfiguration.java | 16 +- .../java/org/apache/oozie/util/JobUtils.java | 4 +- .../org/apache/oozie/util/JobsFilterUtils.java | 4 + .../oozie/util/MetricsInstrumentation.java | 13 + .../apache/oozie/util/NamedThreadFactory.java | 38 + .../apache/oozie/util/PriorityDelayQueue.java | 7 +- .../oozie/util/TimestampedMessageParser.java | 3 +- .../java/org/apache/oozie/util/XCallable.java | 16 +- .../org/apache/oozie/workflow/WorkflowApp.java | 4 +- .../apache/oozie/workflow/WorkflowInstance.java | 40 +- .../org/apache/oozie/workflow/WorkflowLib.java | 18 +- .../oozie/workflow/lite/ActionNodeDef.java | 5 +- .../oozie/workflow/lite/ControlNodeHandler.java | 6 + .../workflow/lite/LiteWorkflowAppParser.java | 382 +---- .../workflow/lite/LiteWorkflowValidator.java | 351 +++++ .../org/apache/oozie/workflow/lite/NodeDef.java | 87 +- .../apache/oozie/workflow/lite/NodeHandler.java | 28 +- core/src/main/resources/oozie-default.xml | 71 +- .../org/apache/hadoop/examples/SleepJob.java | 4 +- .../action/hadoop/ActionExecutorTestCase.java | 2 - .../oozie/action/hadoop/TestDistcpMain.java | 6 +- .../action/hadoop/TestHCatCredentials.java | 258 ++++ .../action/hadoop/TestJavaActionExecutor.java | 49 +- .../action/hadoop/TestShellActionExecutor.java | 67 +- .../oozie/TestSubWorkflowActionExecutor.java | 83 +- .../org/apache/oozie/client/TestOozieCLI.java | 35 +- .../oozie/client/TestWorkflowXClient.java | 4 +- .../bundle/TestBundleChangeXCommand.java | 5 +- .../bundle/TestBundleSubmitXCommand.java | 44 +- .../coord/TestAbandonedCoordChecker.java | 17 +- .../TestCoordActionInputCheckXCommand.java | 4 +- .../coord/TestCoordActionsKillXCommand.java | 1 + .../command/coord/TestCoordChangeXCommand.java | 3 + .../command/coord/TestCoordRerunXCommand.java | 50 +- .../command/coord/TestCoordSubmitXCommand.java | 2 +- .../command/wf/TestActionCheckXCommand.java | 1 - .../command/wf/TestActionStartXCommand.java | 4 - .../oozie/command/wf/TestActionUserRetry.java | 215 +++ .../wf/TestForkedActionStartXCommand.java | 77 - .../oozie/command/wf/TestSignalXCommand.java | 2 +- .../oozie/command/wf/TestSubmitXCommand.java | 84 +- .../command/wf/TestWorkflowKillXCommand.java | 19 + .../wf/TestWorkflowNotificationXCommand.java | 67 + .../apache/oozie/event/TestEventGeneration.java | 20 +- .../jpa/TestBundleJobInfoGetJPAExecutor.java | 110 ++ .../jpa/TestBundleJobQueryExecutor.java | 2 +- .../jpa/TestCoordJobInfoGetJPAExecutor.java | 136 +- .../jpa/TestWorkflowsJobGetJPAExecutor.java | 41 + .../org/apache/oozie/lock/TestMemoryLocks.java | 276 ++-- .../oozie/service/TestActionConfFileType.java | 52 + .../oozie/service/TestAuthorizationService.java | 5 +- .../oozie/service/TestConfigurationService.java | 2 + ...TestHAPartitionDependencyManagerService.java | 1 + .../service/TestHadoopAccessorService.java | 4 + .../oozie/service/TestPauseTransitService.java | 6 +- .../oozie/service/TestRecoveryService.java | 35 +- .../oozie/service/TestShareLibService.java | 29 +- .../oozie/service/TestZKLocksService.java | 115 +- .../apache/oozie/service/TestZKUUIDService.java | 30 +- .../apache/oozie/servlet/TestV2SLAServlet.java | 6 +- .../apache/oozie/store/TestDBWorkflowStore.java | 1 - .../org/apache/oozie/test/XDataTestCase.java | 3 +- .../org/apache/oozie/test/XHCatTestCase.java | 1 + .../java/org/apache/oozie/test/XTestCase.java | 87 +- .../oozie/util/TestJaasConfiguration.java | 19 +- .../oozie/util/TestMetricsInstrumentation.java | 65 + .../util/TestTimestampedMessageParser.java | 4 +- .../lite/TestLiteWorkflowAppParser.java | 112 +- core/src/test/resources/wf-long.xml | 1456 ++++++++++++++++++ distro/pom.xml | 19 +- distro/src/main/bin/oozie-setup.ps1 | 6 +- distro/src/main/bin/oozie-setup.sh | 26 +- distro/src/main/tomcat/ssl-server.xml | 2 +- docs/pom.xml | 4 +- .../src/site/twiki/AG_ActionConfiguration.twiki | 6 +- docs/src/site/twiki/AG_Install.twiki | 104 +- .../site/twiki/CoordinatorFunctionalSpec.twiki | 331 ++++ docs/src/site/twiki/DG_CommandLineTool.twiki | 36 +- docs/src/site/twiki/DG_CoordinatorRerun.twiki | 2 +- docs/src/site/twiki/DG_QuickStart.twiki | 10 +- docs/src/site/twiki/DG_SLAMonitoring.twiki | 26 +- .../site/twiki/DG_ShellActionExtension.twiki | 68 + .../site/twiki/DG_SparkActionExtension.twiki | 60 +- .../site/twiki/DG_SqoopActionExtension.twiki | 2 +- docs/src/site/twiki/ENG_Building.twiki | 8 +- docs/src/site/twiki/ENG_MiniOozie.twiki | 2 +- docs/src/site/twiki/WebServicesAPI.twiki | 12 +- .../src/site/twiki/WorkflowFunctionalSpec.twiki | 18 +- examples/pom.xml | 10 +- .../main/apps/coord-input-logic/coordinator.xml | 79 + .../main/apps/coord-input-logic/job.properties | 25 + .../main/apps/coord-input-logic/workflow.xml | 61 + examples/src/main/apps/distcp/workflow.xml | 2 + examples/src/main/apps/hive/script.q | 1 + .../oozie/example/TestLocalOozieExample.java | 1 - login/pom.xml | 4 +- minitest/pom.xml | 4 +- .../org/apache/oozie/test/WorkflowTest.java | 1 - pom.xml | 149 +- release-log.txt | 97 +- sharelib/distcp/pom.xml | 4 +- .../apache/oozie/action/hadoop/DistcpMain.java | 9 +- sharelib/hcatalog/pom.xml | 4 +- sharelib/hive/pom.xml | 9 +- .../apache/oozie/action/hadoop/HiveMain.java | 22 +- .../action/hadoop/TestHiveActionExecutor.java | 15 +- .../oozie/action/hadoop/TestHiveMain.java | 8 +- sharelib/hive2/pom.xml | 9 +- .../apache/oozie/action/hadoop/Hive2Main.java | 3 +- .../action/hadoop/TestHive2ActionExecutor.java | 51 +- sharelib/oozie/pom.xml | 4 +- .../apache/oozie/action/hadoop/JavaMain.java | 2 + .../oozie/action/hadoop/LauncherMain.java | 103 +- .../oozie/action/hadoop/LauncherMapper.java | 44 +- .../oozie/action/hadoop/LauncherURIHandler.java | 6 +- .../action/hadoop/OozieActionConfigurator.java | 2 +- .../hadoop/OozieLauncherOutputCommitter.java | 65 + .../hadoop/OozieLauncherOutputFormat.java | 48 + .../apache/oozie/action/hadoop/ShellMain.java | 55 +- sharelib/pig/pom.xml | 4 +- .../org/apache/oozie/action/hadoop/PigMain.java | 15 +- .../oozie/action/hadoop/PigMainWithOldAPI.java | 6 +- .../action/hadoop/TestPigActionExecutor.java | 1 - .../apache/oozie/action/hadoop/TestPigMain.java | 1 - sharelib/pom.xml | 4 +- sharelib/spark/pom.xml | 24 +- .../apache/oozie/action/hadoop/SparkMain.java | 235 +-- sharelib/sqoop/pom.xml | 8 +- .../apache/oozie/action/hadoop/SqoopMain.java | 21 +- .../action/hadoop/TestSqoopActionExecutor.java | 22 +- sharelib/streaming/pom.xml | 4 +- .../hadoop/TestMapReduceActionExecutor.java | 2 +- src/main/assemblies/distro.xml | 8 - src/main/assemblies/examples.xml | 4 + tools/pom.xml | 10 +- .../java/org/apache/oozie/tools/OozieDBCLI.java | 3 +- .../apache/oozie/tools/OozieDBExportCLI.java | 200 +++ .../apache/oozie/tools/OozieDBImportCLI.java | 214 +++ .../apache/oozie/tools/OozieSharelibCLI.java | 2 +- .../org/apache/oozie/tools/TestDBLoadDump.java | 115 ++ .../src/test/resources/dumpData/ooziedb_ac.json | 3 + .../test/resources/dumpData/ooziedb_bna.json | 0 .../test/resources/dumpData/ooziedb_bnj.json | 0 .../src/test/resources/dumpData/ooziedb_ca.json | 3 + .../src/test/resources/dumpData/ooziedb_cj.json | 3 + .../test/resources/dumpData/ooziedb_slareg.json | 0 .../test/resources/dumpData/ooziedb_slasum.json | 0 .../resources/dumpData/ooziedb_sysinfo.json | 1 + .../src/test/resources/dumpData/ooziedb_wf.json | 1 + webapp/pom.xml | 6 +- .../webapp/console/sla/js/oozie-sla-table.js | 21 +- webapp/src/main/webapp/oozie-console.js | 3 +- workflowgenerator/README.txt | 138 -- workflowgenerator/pom.xml | 185 --- .../src/main/assemblies/workflowgenerator.xml | 30 - .../client/OozieDiagramController.java | 452 ------ .../client/OozieWorkflowGenerator.java | 765 --------- .../client/property/Property.java | 75 - .../client/property/PropertyTable.java | 620 -------- .../client/property/PropertyTableFactory.java | 137 -- .../property/action/EmailPropertyTable.java | 114 -- .../client/property/action/FSPropertyTable.java | 477 ------ .../property/action/JavaPropertyTable.java | 189 --- .../property/action/MapReducePropertyTable.java | 156 -- .../property/action/PigPropertyTable.java | 168 -- .../property/action/PipesPropertyTable.java | 207 --- .../property/action/SSHPropertyTable.java | 234 --- .../property/action/ShellPropertyTable.java | 190 --- .../property/action/StreamingPropertyTable.java | 206 --- .../property/action/SubWFPropertyTable.java | 140 -- .../property/control/DecisionPropertyTable.java | 423 ----- .../property/control/EndPropertyTable.java | 67 - .../property/control/ForkPropertyTable.java | 237 --- .../property/control/JoinPropertyTable.java | 72 - .../property/control/KillPropertyTable.java | 77 - .../property/control/StartPropertyTable.java | 73 - .../property/control/WrkflowPropertyTable.java | 169 -- .../client/widget/NodeWidget.java | 272 ---- .../client/widget/action/EmailActionWidget.java | 48 - .../client/widget/action/FSActionWidget.java | 48 - .../client/widget/action/JavaActionWidget.java | 49 - .../widget/action/MapReduceActionWidget.java | 48 - .../client/widget/action/PigActionWidget.java | 49 - .../client/widget/action/PipesActionWidget.java | 48 - .../client/widget/action/SSHActionWidget.java | 48 - .../client/widget/action/ShellActionWidget.java | 48 - .../widget/action/StreamingActionWidget.java | 48 - .../client/widget/action/SubWFActionWidget.java | 48 - .../widget/control/DecisionNodeWidget.java | 49 - .../client/widget/control/EndNodeWidget.java | 56 - .../client/widget/control/ForkNodeWidget.java | 50 - .../client/widget/control/JoinNodeWidget.java | 47 - .../client/widget/control/KillNodeWidget.java | 42 - .../client/widget/control/StartNodeWidget.java | 69 - .../workflowgenerator/workflowgenerator.gwt.xml | 43 - .../src/main/resources/img/action.png | Bin 889 -> 0 bytes .../src/main/resources/img/add-btn.png | Bin 1656 -> 0 bytes .../src/main/resources/img/decision.png | Bin 1042 -> 0 bytes .../src/main/resources/img/del-btn.png | Bin 1521 -> 0 bytes .../src/main/resources/img/distcp-action.png | Bin 1584 -> 0 bytes .../src/main/resources/img/email-action.png | Bin 1454 -> 0 bytes .../src/main/resources/img/end.png | Bin 1246 -> 0 bytes .../src/main/resources/img/fork-shape.png | Bin 1591 -> 0 bytes .../src/main/resources/img/fork.png | Bin 2078 -> 0 bytes .../src/main/resources/img/fs-action.png | Bin 1310 -> 0 bytes .../src/main/resources/img/hive-action.png | Bin 1401 -> 0 bytes .../src/main/resources/img/java-action.png | Bin 1428 -> 0 bytes .../src/main/resources/img/join-shape.png | Bin 1609 -> 0 bytes .../src/main/resources/img/join.png | Bin 2141 -> 0 bytes .../src/main/resources/img/kill.png | Bin 2439 -> 0 bytes .../src/main/resources/img/mr-action.png | Bin 1335 -> 0 bytes .../src/main/resources/img/pig-action.png | Bin 1387 -> 0 bytes .../src/main/resources/img/pipes-action.png | Bin 1610 -> 0 bytes .../src/main/resources/img/shell-action.png | Bin 1612 -> 0 bytes .../src/main/resources/img/ssh-action.png | Bin 1346 -> 0 bytes .../src/main/resources/img/start-shape.png | Bin 1513 -> 0 bytes .../src/main/resources/img/start.png | Bin 2368 -> 0 bytes .../src/main/resources/img/streaming-action.png | Bin 1566 -> 0 bytes .../src/main/resources/img/subwf-action.png | Bin 1562 -> 0 bytes .../workflowgenerator/workflowgenerator.gwt.xml | 43 - .../src/main/webapp/WEB-INF/web.xml | 31 - .../src/main/webapp/workflowgenerator.css | 200 --- .../src/main/webapp/workflowgenerator.html | 66 - .../client/OozieWorkflowGeneratorTest.java | 35 - .../workflowgeneratorTest-dev.launch | 39 - .../workflowgeneratorTest-prod.launch | 39 - zookeeper-security-tests/pom.xml | 4 +- 318 files changed, 7740 insertions(+), 9420 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/client/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/client/src/main/java/org/apache/oozie/client/XOozieClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java index 12cc016,2aed936..ddb1504 --- a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java @@@ -127,19 -121,7 +121,7 @@@ public class Hive2ActionExecutor extend return conf; } - @Override - protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { - return true; - } - - @Override - protected void getActionData(FileSystem actionFs, WorkflowAction action, Context context) - throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { - super.getActionData(actionFs, action, context); - readExternalChildIDs(action, context); - } - -- /** ++ /** * Return the sharelib name for the action. * * @return returns <code>hive2</code>. http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index dbe7bc1,65996d9..0515d39 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@@ -148,10 -120,18 +148,12 @@@ public class JavaActionExecutor extend protected XLog LOG = XLog.getLog(getClass()); private static final Pattern heapPattern = Pattern.compile("-Xmx(([0-9]+)[mMgG])"); private static final String JAVA_TMP_DIR_SETTINGS = "-Djava.io.tmpdir="; - public static final String CONF_HADOOP_YARN_UBER_MODE = "oozie.action.launcher." + HADOOP_YARN_UBER_MODE; - public static final String HADOOP_JOB_CLASSLOADER = "mapreduce.job.classloader"; - public static final String HADOOP_USER_CLASSPATH_FIRST = "mapreduce.user.classpath.first"; - public static final String OOZIE_CREDENTIALS_SKIP = "oozie.credentials.skip"; + public XConfiguration workflowConf = null; + static { DISALLOWED_PROPERTIES.add(HADOOP_USER); - DISALLOWED_PROPERTIES.add(HADOOP_JOB_TRACKER); DISALLOWED_PROPERTIES.add(HADOOP_NAME_NODE); - DISALLOWED_PROPERTIES.add(HADOOP_JOB_TRACKER_2); DISALLOWED_PROPERTIES.add(HADOOP_YARN_RM); } @@@ -163,13 -143,15 +165,15 @@@ super(type); } - public static List<Class> getCommonLauncherClasses() { - List<Class> classes = new ArrayList<Class>(); - classes.add(LauncherMapper.class); + public static List<Class<?>> getCommonLauncherClasses() { + List<Class<?>> classes = new ArrayList<Class<?>>(); classes.add(OozieLauncherInputFormat.class); + classes.add(LauncherMain.class); + classes.add(OozieLauncherOutputFormat.class); + classes.add(OozieLauncherOutputCommitter.class); - classes.add(LauncherMainHadoopUtils.class); - classes.add(HadoopShims.class); classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher()); + classes.add(LauncherAM.class); + classes.add(LauncherAMCallbackNotifier.class); return classes; } @@@ -1451,104 -1425,114 +1443,105 @@@ @Override public void check(Context context, WorkflowAction action) throws ActionExecutorException { - JobClient jobClient = null; - boolean exception = false; + boolean fallback = false; + LOG = XLog.resetPrefix(LOG); LogUtils.setLogInfo(action); + YarnClient yarnClient = null; try { Element actionXml = XmlUtils.parseXml(action.getConf()); - FileSystem actionFs = context.getAppFileSystem(); JobConf jobConf = createBaseHadoopConf(context, actionXml); - jobClient = createJobClient(context, jobConf); - RunningJob runningJob = getRunningJob(context, action, jobClient); - if (runningJob == null) { - context.setExecutionData(FAILED, null); - throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "JA017", - "Could not lookup launched hadoop Job ID [{0}] which was associated with " + - " action [{1}]. Failing this action!", getActualExternalId(action), action.getId()); + FileSystem actionFs = context.getAppFileSystem(); + yarnClient = createYarnClient(context, jobConf); + FinalApplicationStatus appStatus = null; + try { + ApplicationReport appReport = + yarnClient.getApplicationReport(ConverterUtils.toApplicationId(action.getExternalId())); + YarnApplicationState appState = appReport.getYarnApplicationState(); + if (appState == YarnApplicationState.FAILED || appState == YarnApplicationState.FINISHED + || appState == YarnApplicationState.KILLED) { + appStatus = appReport.getFinalApplicationStatus(); + } + + } catch (Exception ye) { + LOG.debug("Exception occurred while checking Launcher AM status; will try checking action data file instead ", ye); + // Fallback to action data file if we can't find the Launcher AM (maybe it got purged) + fallback = true; } - if (runningJob.isComplete()) { + if (appStatus != null || fallback) { Path actionDir = context.getActionDir(); - String newId = null; // load sequence file into object Map<String, String> actionData = LauncherMapperHelper.getActionData(actionFs, actionDir, jobConf); - if (actionData.containsKey(LauncherMapper.ACTION_DATA_NEW_ID)) { - newId = actionData.get(LauncherMapper.ACTION_DATA_NEW_ID); - String launcherId = action.getExternalId(); - runningJob = jobClient.getJob(JobID.forName(newId)); - if (runningJob == null) { - context.setExternalStatus(FAILED); + if (fallback) { + String finalStatus = actionData.get(LauncherAM.ACTION_DATA_FINAL_STATUS); + if (finalStatus != null) { + appStatus = FinalApplicationStatus.valueOf(finalStatus); + } else { + context.setExecutionData(FAILED, null); throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "JA017", - "Unknown hadoop job [{0}] associated with action [{1}]. Failing this action!", newId, - action.getId()); + "Unknown hadoop job [{0}] associated with action [{1}] and couldn't determine status from" + + " action data. Failing this action!", action.getExternalId(), action.getId()); } - context.setExternalChildIDs(newId); - LOG.info(XLog.STD, "External ID swap, old ID [{0}] new ID [{1}]", launcherId, - newId); } - else { - String externalIDs = actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS); - if (externalIDs != null) { - context.setExternalChildIDs(externalIDs); - LOG.info(XLog.STD, "Hadoop Jobs launched : [{0}]", externalIDs); + + String externalID = actionData.get(LauncherAM.ACTION_DATA_NEW_ID); // MapReduce was launched + if (externalID != null) { + context.setExternalChildIDs(externalID); + LOG.info(XLog.STD, "Hadoop Job was launched : [{0}]", externalID); + } + + // Multiple child IDs - Pig or Hive action + String externalIDs = actionData.get(LauncherAM.ACTION_DATA_EXTERNAL_CHILD_IDS); + if (externalIDs != null) { + context.setExternalChildIDs(externalIDs); + LOG.info(XLog.STD, "External Child IDs : [{0}]", externalIDs); ++ + } + + LOG.info(XLog.STD, "action completed, external ID [{0}]", action.getExternalId()); + context.setExecutionData(appStatus.toString(), null); + if (appStatus == FinalApplicationStatus.SUCCEEDED) { + if (getCaptureOutput(action) && LauncherMapperHelper.hasOutputData(actionData)) { + context.setExecutionData(SUCCEEDED, PropertiesUtils.stringToProperties(actionData + .get(LauncherAM.ACTION_DATA_OUTPUT_PROPS))); + LOG.info(XLog.STD, "action produced output"); } - else if (LauncherMapperHelper.hasOutputData(actionData)) { - // Load stored Hadoop jobs ids and promote them as external child ids - // This is for jobs launched with older release during upgrade to Oozie 4.3 - Properties props = PropertiesUtils.stringToProperties(actionData - .get(LauncherMapper.ACTION_DATA_OUTPUT_PROPS)); - if (props.get(LauncherMain.HADOOP_JOBS) != null) { - externalIDs = (String) props.get(LauncherMain.HADOOP_JOBS); - context.setExternalChildIDs(externalIDs); - LOG.info(XLog.STD, "Hadoop Jobs launched : [{0}]", externalIDs); - } + else { + context.setExecutionData(SUCCEEDED, null); } - } - if (runningJob.isComplete()) { - // fetching action output and stats for the Map-Reduce action. - if (newId != null) { - actionData = LauncherMapperHelper.getActionData(actionFs, context.getActionDir(), jobConf); + if (LauncherMapperHelper.hasStatsData(actionData)) { + context.setExecutionStats(actionData.get(LauncherAM.ACTION_DATA_STATS)); + LOG.info(XLog.STD, "action produced stats"); } - LOG.info(XLog.STD, "action completed, external ID [{0}]", - action.getExternalId()); - if (LauncherMapperHelper.isMainSuccessful(runningJob)) { - if (getCaptureOutput(action) && LauncherMapperHelper.hasOutputData(actionData)) { - context.setExecutionData(SUCCEEDED, PropertiesUtils.stringToProperties(actionData - .get(LauncherMapper.ACTION_DATA_OUTPUT_PROPS))); - LOG.info(XLog.STD, "action produced output"); + getActionData(actionFs, action, context); + } + else { + String errorReason; + if (actionData.containsKey(LauncherAM.ACTION_DATA_ERROR_PROPS)) { + Properties props = PropertiesUtils.stringToProperties(actionData + .get(LauncherAM.ACTION_DATA_ERROR_PROPS)); + String errorCode = props.getProperty("error.code"); + if ("0".equals(errorCode)) { + errorCode = "JA018"; } - else { - context.setExecutionData(SUCCEEDED, null); + if ("-1".equals(errorCode)) { + errorCode = "JA019"; } - if (LauncherMapperHelper.hasStatsData(actionData)) { - context.setExecutionStats(actionData.get(LauncherMapper.ACTION_DATA_STATS)); - LOG.info(XLog.STD, "action produced stats"); + errorReason = props.getProperty("error.reason"); + LOG.warn("Launcher ERROR, reason: {0}", errorReason); + String exMsg = props.getProperty("exception.message"); + String errorInfo = (exMsg != null) ? exMsg : errorReason; + context.setErrorInfo(errorCode, errorInfo); + String exStackTrace = props.getProperty("exception.stacktrace"); + if (exMsg != null) { + LOG.warn("Launcher exception: {0}{E}{1}", exMsg, exStackTrace); } - getActionData(actionFs, runningJob, action, context); } else { - String errorReason; - if (actionData.containsKey(LauncherMapper.ACTION_DATA_ERROR_PROPS)) { - Properties props = PropertiesUtils.stringToProperties(actionData - .get(LauncherMapper.ACTION_DATA_ERROR_PROPS)); - String errorCode = props.getProperty("error.code"); - if ("0".equals(errorCode)) { - errorCode = "JA018"; - } - if ("-1".equals(errorCode)) { - errorCode = "JA019"; - } - errorReason = props.getProperty("error.reason"); - LOG.warn("Launcher ERROR, reason: {0}", errorReason); - String exMsg = props.getProperty("exception.message"); - String errorInfo = (exMsg != null) ? exMsg : errorReason; - context.setErrorInfo(errorCode, errorInfo); - String exStackTrace = props.getProperty("exception.stacktrace"); - if (exMsg != null) { - LOG.warn("Launcher exception: {0}{E}{1}", exMsg, exStackTrace); - } - } - else { - errorReason = XLog.format("LauncherMapper died, check Hadoop LOG for job [{0}:{1}]", action - .getTrackerUri(), action.getExternalId()); - LOG.warn(errorReason); - } - context.setExecutionData(FAILED_KILLED, null); + errorReason = XLog.format("Launcher AM died, check Hadoop LOG for job [{0}:{1}]", action + .getTrackerUri(), action.getExternalId()); + LOG.warn(errorReason); } - } - else { - context.setExternalStatus("RUNNING"); - LOG.info(XLog.STD, "checking action, hadoop job ID [{0}] status [RUNNING]", - runningJob.getID()); + context.setExecutionData(FAILED_KILLED, null); } } else { http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java index bb58ad5,9609fdc..e2a667e --- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java @@@ -38,11 -39,12 +38,12 @@@ import org.apache.hadoop.fs.FileSystem import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RunningJob; -import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.security.UserGroupInformation; import org.apache.oozie.client.OozieClient; + import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.Services; @@@ -51,9 -53,10 +52,11 @@@ import org.apache.oozie.service.UserGro import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.PropertiesUtils; +// TODO: we're no longer using Launcher Mapper -- give this class a better name public class LauncherMapperHelper { + public static final String OOZIE_ACTION_YARN_TAG = "oozie.action.yarn.tag"; + public static String getRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId) throws HadoopAccessorException, IOException { String jobId = null; @@@ -145,8 -148,21 +148,9 @@@ launcherConf.setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true); } - FileSystem fs = - Services.get().get(HadoopAccessorService.class).createFileSystem(launcherConf.get("user.name"), - actionDir.toUri(), launcherConf); - fs.mkdirs(actionDir); - - OutputStream os = fs.create(new Path(actionDir, LauncherMapper.ACTION_CONF_XML)); - try { - actionConf.writeXml(os); - } finally { - IOUtils.closeSafely(os); - } - launcherConf.setInputFormat(OozieLauncherInputFormat.class); - launcherConf.set("mapred.output.dir", new Path(actionDir, "output").toString()); + launcherConf.setOutputFormat(OozieLauncherOutputFormat.class); + launcherConf.setOutputCommitter(OozieLauncherOutputCommitter.class); } public static void setupYarnRestartHandling(JobConf launcherJobConf, Configuration actionConf, String launcherTag, @@@ -157,10 -173,10 +161,10 @@@ String tag = getTag(launcherTag); // keeping the oozie.child.mapreduce.job.tags instead of mapreduce.job.tags to avoid killing launcher itself. // mapreduce.job.tags should only go to child job launch by launcher. - actionConf.set(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS, tag); + actionConf.set(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS, tag); } - private static String getTag(String launcherTag) throws NoSuchAlgorithmException { + public static String getTag(String launcherTag) throws NoSuchAlgorithmException { MessageDigest digest = MessageDigest.getInstance("MD5"); digest.update(launcherTag.getBytes(), 0, launcherTag.length()); String md5 = "oozie-" + new BigInteger(1, digest.digest()).toString(16); http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java index 019c4d9,c36a675..89f30f1 --- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java @@@ -314,80 -334,19 +314,86 @@@ public class MapReduceActionExecutor ex } @Override - protected RunningJob getRunningJob(Context context, WorkflowAction action, JobClient jobClient) throws Exception{ + protected void injectCallback(Context context, Configuration conf) { + // add callback for the MapReduce job + String callback = context.getCallbackUrl("$jobStatus"); + if (conf.get("job.end.notification.url") != null) { + LOG.warn("Overriding the action job end notification URI"); + } + conf.set("job.end.notification.url", callback); + + super.injectCallback(context, conf); + } - RunningJob runningJob; - String jobId = getActualExternalId(action); + @Override + public void check(Context context, WorkflowAction action) throws ActionExecutorException { + Map<String, String> actionData = Collections.emptyMap(); + JobConf jobConf = null; + + try { + FileSystem actionFs = context.getAppFileSystem(); + Element actionXml = XmlUtils.parseXml(action.getConf()); + jobConf = createBaseHadoopConf(context, actionXml); + Path actionDir = context.getActionDir(); + actionData = LauncherMapperHelper.getActionData(actionFs, actionDir, jobConf); + } catch (Exception e) { + LOG.warn("Exception in check(). Message[{0}]", e.getMessage(), e); + throw convertException(e); + } - runningJob = jobClient.getJob(JobID.forName(jobId)); + final String newId = actionData.get(LauncherMapper.ACTION_DATA_NEW_ID); - return runningJob; + // check the Hadoop job if newID is defined (which should be the case here) - otherwise perform the normal check() + if (newId != null) { + boolean jobCompleted; + JobClient jobClient = null; + boolean exception = false; + + try { + jobClient = createJobClient(context, jobConf); + RunningJob runningJob = jobClient.getJob(JobID.forName(newId)); + + if (runningJob == null) { + context.setExternalStatus(FAILED); + throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "JA017", + "Unknown hadoop job [{0}] associated with action [{1}]. Failing this action!", newId, + action.getId()); + } + + jobCompleted = runningJob.isComplete(); + } catch (Exception e) { + LOG.warn("Exception in check(). Message[{0}]", e.getMessage(), e); + exception = true; + throw convertException(e); + } finally { + if (jobClient != null) { + try { + jobClient.close(); + } catch (Exception e) { + if (exception) { + LOG.error("JobClient error (not re-throwing due to a previous error): ", e); + } else { + throw convertException(e); + } + } + } + } + + // run original check() if the MR action is completed or there are errors - otherwise mark it as RUNNING + if (jobCompleted || (!jobCompleted && actionData.containsKey(LauncherMapper.ACTION_DATA_ERROR_PROPS))) { + super.check(context, action); + } else { + context.setExternalStatus(RUNNING); + context.setExternalChildIDs(newId); + } + } else { + super.check(context, action); + } } + + @Override + void injectActionCallback(Context context, Configuration actionConf) { + injectCallback(context, actionConf); + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index 5f33bb2,8e8d7d3..32dadf0 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@@ -34,12 -28,16 +28,15 @@@ import org.apache.oozie.service.Configu import org.apache.oozie.service.Services; import org.apache.oozie.service.SparkConfigurationService; import org.jdom.Element; - import org.jdom.JDOMException; import org.jdom.Namespace; + import java.util.ArrayList; + import java.util.List; + import java.util.Map; + public class SparkActionExecutor extends JavaActionExecutor { public static final String SPARK_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.SparkMain"; - public static final String TASK_USER_PRECEDENCE = "mapreduce.task.classpath.user.precedence"; // hadoop-2 - public static final String TASK_USER_CLASSPATH_PRECEDENCE = "mapreduce.user.classpath.first"; // hadoop-1 + public static final String TASK_USER_PRECEDENCE = "mapreduce.task.classpath.user.precedence"; public static final String SPARK_MASTER = "oozie.spark.master"; public static final String SPARK_MODE = "oozie.spark.mode"; public static final String SPARK_OPTS = "oozie.spark.spark-opts"; http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java index c3a09ac,6cee32a..b0fc83a --- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java @@@ -18,15 -18,7 +18,13 @@@ package org.apache.oozie.action.hadoop; +import java.io.IOException; +import java.io.StringReader; - import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.StringTokenizer; + import org.apache.hadoop.conf.Configuration; - import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.JobClient; @@@ -35,14 -27,18 +33,12 @@@ import org.apache.hadoop.mapred.JobID import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; - import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.util.XConfiguration; - import org.apache.oozie.util.XmlUtils; import org.apache.oozie.util.XLog; + import org.apache.oozie.util.XmlUtils; import org.jdom.Element; - import org.jdom.JDOMException; import org.jdom.Namespace; -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; -import java.util.StringTokenizer; - public class SqoopActionExecutor extends JavaActionExecutor { public static final String OOZIE_ACTION_EXTERNAL_STATS_WRITE = "oozie.action.external.stats.write"; @@@ -229,26 -225,6 +225,7 @@@ } } - /** - * Get the stats and external child IDs - * @param actionFs the FileSystem object - * @param action the Workflow action - * @param context executor context - * - */ - @Override - protected void getActionData(FileSystem actionFs, WorkflowAction action, Context context) - throws HadoopAccessorException, JDOMException, IOException, URISyntaxException{ - super.getActionData(actionFs, action, context); - readExternalChildIDs(action, context); - } - - @Override - protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { - return true; - } - + /** * Return the sharelib name for the action. * http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java index 64fa89a,cc61d3d..26deda8 --- a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java @@@ -41,15 -41,11 +41,12 @@@ public class SubmitMRXCommand extends S static { SKIPPED_CONFS.add(WorkflowAppService.HADOOP_USER); - SKIPPED_CONFS.add(XOozieClient.JT); + SKIPPED_CONFS.add(XOozieClient.RM); SKIPPED_CONFS.add(XOozieClient.NN); - // a brillant mind made a change in Configuration that 'fs.default.name' key gets converted to 'fs.defaultFS' - // in Hadoop 0.23, we need skip that one too, keeping the old one because of Hadoop 1 - SKIPPED_CONFS.add(XOozieClient.NN_2); DEPRECATE_MAP.put(XOozieClient.NN, XOozieClient.NN_2); - DEPRECATE_MAP.put(XOozieClient.JT, XOozieClient.JT_2); + DEPRECATE_MAP.put(XOozieClient.RM, "yarn.resourcemanager.address"); + DEPRECATE_MAP.put(XOozieClient.NN, "fs.defaultFS"); DEPRECATE_MAP.put(WorkflowAppService.HADOOP_USER, "mapreduce.job.user.name"); } http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java index 5845e17,5377127..0824503 --- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java @@@ -30,15 -29,9 +30,16 @@@ import org.apache.hadoop.net.NetUtils import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Records; import org.apache.oozie.ErrorCode; +import org.apache.oozie.action.ActionExecutor; import org.apache.oozie.action.hadoop.JavaActionExecutor; + import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.ParamChecker; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XLog; http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/service/ShareLibService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/java/org/apache/oozie/util/JobUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --cc core/src/main/resources/oozie-default.xml index 5f4645c,e71ebe3..59b359b --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@@ -1782,6 -1790,42 +1790,17 @@@ will be the requeue interval for the ac </property> <property> - <name>oozie.action.launcher.mapreduce.job.ubertask.enable</name> - <value>true</value> - <description> - Enables Uber Mode for the launcher job in YARN/Hadoop 2 (no effect in Hadoop 1) for all action types by default. - This can be overridden on a per-action-type basis by setting - oozie.action.#action-type#.launcher.mapreduce.job.ubertask.enable in oozie-site.xml (where #action-type# is the action - type; for example, "pig"). And that can be overridden on a per-action basis by setting - oozie.launcher.mapreduce.job.ubertask.enable in an action's configuration section in a workflow. In summary, the - priority is this: - 1. action's configuration section in a workflow - 2. oozie.action.#action-type#.launcher.mapreduce.job.ubertask.enable in oozie-site - 3. oozie.action.launcher.mapreduce.job.ubertask.enable in oozie-site - </description> - </property> - - <property> - <name>oozie.action.shell.launcher.mapreduce.job.ubertask.enable</name> - <value>false</value> - <description> - The Shell action may have issues with the $PATH environment when using Uber Mode, and so Uber Mode is disabled by - default for it. See oozie.action.launcher.mapreduce.job.ubertask.enable - </description> - </property> - - <property> + <name>oozie.action.spark.setup.hadoop.conf.dir</name> + <value>false</value> + <description> + Oozie action.xml (oozie.action.conf.xml) contains all the hadoop configuration and user provided configurations. + This property will allow users to copy Oozie action.xml as hadoop *-site configurations files. The advantage is, + user need not to manage these files into spark sharelib. If user wants to manage the hadoop configurations + themselves, it should should disable it. + </description> + </property> + + <property> <name>oozie.action.shell.setup.hadoop.conf.dir</name> <value>false</value> <description> http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 8adc606,75301db..5d8bf34 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@@ -490,7 -543,73 +494,7 @@@ public class TestJavaActionExecutor ext assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); } - public void testChildKill() throws Exception { - if (HadoopShims.isYARN()) { - final JobConf clusterConf = createJobConf(); - FileSystem fileSystem = FileSystem.get(clusterConf); - Path confFile = new Path("/tmp/cluster-conf.xml"); - OutputStream out = fileSystem.create(confFile); - clusterConf.writeXml(out); - out.close(); - String confFileName = fileSystem.makeQualified(confFile).toString() + "#core-site.xml"; - final String actionXml = "<java>" + - "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + - "<name-node>" + getNameNodeUri() + "</name-node>" + - "<main-class> " + SleepJob.class.getName() + " </main-class>" + - "<arg>-mt</arg>" + - "<arg>300000</arg>" + - "<archive>" + confFileName + "</archive>" + - "</java>"; - final Context context = createContext(actionXml, null); - final RunningJob runningJob = submitAction(context); - waitFor(60 * 1000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return runningJob.getJobStatus().getRunState() == 1; - } - }); - assertFalse(runningJob.isComplete()); - Thread.sleep(15000); - UserGroupInformationService ugiService = Services.get(). - get(UserGroupInformationService.class); - - UserGroupInformation ugi = ugiService.getProxyUser(getTestUser()); - ugi.doAs(new PrivilegedExceptionAction<Object>() { - @Override - public Void run() throws Exception { - JavaActionExecutor ae = new JavaActionExecutor(); - ae.kill(context, context.getAction()); - - WorkflowJob wfJob = context.getWorkflow(); - Configuration conf = null; - if (wfJob.getConf() != null) { - conf = new XConfiguration(new StringReader(wfJob.getConf())); - } - String launcherTag = LauncherMapperHelper.getActionYarnTag(conf, wfJob.getParentId(), context.getAction()); - Configuration jobConf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml)); - jobConf.set(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS, LauncherMapperHelper.getTag(launcherTag)); - jobConf.setLong(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME, - context.getAction().getStartTime().getTime()); - Set<String> childSet = LauncherMainHadoopUtils.getChildJobs(jobConf); - assertEquals(1, childSet.size()); - - JobClient jobClient = new JobClient(clusterConf); - for (String jobId : childSet) { - RunningJob childJob = jobClient.getJob(jobId); - assertEquals(JobStatus.State.KILLED.getValue(), childJob.getJobStatus().getRunState()); - } - assertTrue(ae.isCompleted(context.getAction().getExternalStatus())); - return null; - } - }); - - assertEquals(WorkflowAction.Status.DONE, context.getAction().getStatus()); - assertEquals("KILLED", context.getAction().getExternalStatus()); - assertFalse(runningJob.isSuccessful()); - } - } - -- public void testExceptionSubmitException() throws Exception { ++ public void testExceptionSubmitException() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/docs/src/site/twiki/DG_QuickStart.twiki ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/docs/src/site/twiki/ENG_Building.twiki ---------------------------------------------------------------------- diff --cc docs/src/site/twiki/ENG_Building.twiki index cce219e,f6c88d6..a817b03 --- a/docs/src/site/twiki/ENG_Building.twiki +++ b/docs/src/site/twiki/ENG_Building.twiki @@@ -212,10 -212,12 +212,10 @@@ $ bin/mkdistro.sh [-DskipTests Running =mkdistro.sh= will create the binary distribution of Oozie. The following options are available to customise the versions of the dependencies: -Puber - Bundle required hadoop and hcatalog libraries in oozie war --P<profile> - default hadoop-2. Valid are hadoop-1, hadoop-2 or hadoop-3. Choose the correct hadoop -profile depending on the hadoop version used. --Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.4.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3 +-Dhadoop.version=<version> - default 2.4.0 -Dhadoop.auth.version=<version> - defaults to hadoop version -Ddistcp.version=<version> - defaults to hadoop version - -Dpig.version=<version> - default 0.12.1 + -Dpig.version=<version> - default 0.16.0 -Dpig.classifier=<classifier> - default none -Dsqoop.version=<version> - default 1.4.3 -Dsqoop.classifier=<classifier> - default hadoop100 http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/pom.xml ---------------------------------------------------------------------- diff --cc pom.xml index 9d86e71,704a2ee..ef22b39 --- a/pom.xml +++ b/pom.xml @@@ -77,32 -79,40 +79,37 @@@ <oozie.test.config.file>${oozie.test.default.config.file}</oozie.test.config.file> <hadoop.version>2.4.0</hadoop.version> - <hadoop.majorversion>2</hadoop.majorversion> - <hadooplib.version>hadoop-${hadoop.majorversion}-${project.version}</hadooplib.version> - <oozie.hadoop-utils.version>hadoop-${hadoop.majorversion}-${project.version}</oozie.hadoop-utils.version> <hbase.version>0.94.27</hbase.version> - <dropwizard.metrics.version>3.1.0</dropwizard.metrics.version> + <dropwizard.metrics.version>3.1.2</dropwizard.metrics.version> <clover.license>/home/jenkins/tools/clover/latest/lib/clover.license</clover.license> + <!-- Sharelib component versions --> <hive.version>0.13.1</hive.version> + <hive.jline.version>0.9.94</hive.jline.version> - <pig.version>0.16.0</pig.version> - <pig.classifier></pig.classifier> + <pig.version>0.12.1</pig.version> + <pig.classifier>h2</pig.classifier> <sqoop.version>1.4.3</sqoop.version> <spark.version>1.6.1</spark.version> <spark.guava.version>14.0.1</spark.guava.version> + <spark.scala.binary.version>2.10</spark.scala.binary.version> - <sqoop.classifier>hadoop100</sqoop.classifier> + <sqoop.classifier>hadoop200</sqoop.classifier> <streaming.version>${hadoop.version}</streaming.version> - <distcp.version>${hadoop.version}</distcp.version> <hadoop.auth.version>${hadoop.version}</hadoop.auth.version> <!-- Tomcat version --> <tomcat.version>6.0.44</tomcat.version> - <openjpa.version>2.2.2</openjpa.version> + <openjpa.version>2.4.1</openjpa.version> <xerces.version>2.10.0</xerces.version> <curator.version>2.5.0</curator.version> - <jackson.version>1.8.8</jackson.version> + <jackson.version>1.9.13</jackson.version> <log4j.version>1.2.17</log4j.version> + <activemq.version>5.13.3</activemq.version> + <httpcore.version>4.3.3</httpcore.version> + <httpclient.version>4.3.6</httpclient.version> </properties> <modules> @@@ -1562,8 -1636,8 +1613,9 @@@ <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>findbugs-maven-plugin</artifactId> + <version>3.0.1</version> <configuration> + <excludeSubProjects>false</excludeSubProjects> <xmlOutput>true</xmlOutput> <findbugsXmlOutput>true</findbugsXmlOutput> <findbugsXmlWithMessages>true</findbugsXmlWithMessages> @@@ -1728,29 -1878,14 +1831,27 @@@ <activation> <activeByDefault>false</activeByDefault> </activation> - <properties> - <hadoop.version>3.0.0-SNAPSHOT</hadoop.version> - <hadoop.majorversion>3</hadoop.majorversion> - <pig.classifier>h2</pig.classifier> - <sqoop.classifier>hadoop200</sqoop.classifier> - <jackson.version>1.9.13</jackson.version> - </properties> + <modules> + <module>workflowgenerator</module> + </modules> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <fork>true</fork> + <source>1.6</source> + <target>1.6</target> + <compilerArguments> + <Xlint/> + <Xmaxwarns>9999</Xmaxwarns> + </compilerArguments> + </configuration> + </plugin> + </plugins> + </build> - </profile> - <profile> <id>loginServerExample</id> <activation> http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/release-log.txt ---------------------------------------------------------------------- diff --cc release-log.txt index c75911e,13c11df..af44107 --- a/release-log.txt +++ b/release-log.txt @@@ -1,7 -1,100 +1,102 @@@ - -- Oozie 4.3.0 release (trunk - unreleased) + -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2590 OYA: Create basic Oozie Launcher Application Master (rkanter) +OOZIE-2316 Drop support for Hadoop 1 and 0.23 (asasvari via rkanter) + + -- Oozie 4.3.0 release + + OOZIE-2673 Include XSD for shell-action:0.3 in documentation (abhishekbafna via rkanter) + OOZIE-2194 oozie job -kill doesn't work with spark action (abhishekbafna via rohini) + OOZIE-2501 ZK reentrant lock doesn't work for few cases (puru) + OOZIE-2582 Populating external child Ids for action failures (abhishekbafna via rohini) + OOZIE-2678 Oozie job -kill doesn't work with tez jobs (abhishekbafna via rohini) + OOZIE-2676 Make hadoop-2 as the default profile (gezapeti via rkanter) + OOZIE-2487 Temporary workaround for Java 8 Javadoc errors (abhishekbafna via rkanter) + OOZIE-1978 Forkjoin validation code is ridiculously slow in some cases (pbacsko via rkanter) + OOZIE-2682 Oozie test-patch script is not updating Jira with jenkins test report (abhishekbafna via rkanter) + OOZIE-2679 Decrease HttpClient library versions due to Hadoop incompatibility (abhishekbafna via rkanter) + OOZIE-2661 Coordinator Action status not updated when workflow job SUSPENDED (satishsaley via puru) + OOZIE-2273 MiniOozie does not work outside of Oozie (rkanter) + OOZIE-2667 Optimize queries for DB export (gezapeti via rkanter) + OOZIE-1793 Improve find bugs reporting for Oozie (rkanter) + OOZIE-2572 SLA DURATION miss not shown when job is running for longer than expected time (satishsaley via puru) + OOZIE-2525 SchemaChecker fails with NPE (rkanter) + OOZIE-2672 SLA periodic update does not remove job from map if job is removed from database (satishsaley via puru) + OOZIE-2498 Oozie CallerId configuration for downstream components (abhishekbafna via rohini) + OOZIE-2491 oozie acl cannot specify group,it does`t work (abhishekbafna via rohini) + OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini) + OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini) + OOZIE-2588 Support getting credentials for same cluster hcat when credentials config is empty (satishsaley via rohini) + OOZIE-2538 Update HttpClient versions to close security vulnerabilities (abhishekbafna via rkanter) + OOZIE-2037 Add TLSv1.1,TLSv1.2 (rkanter) + OOZIE-2500 -DtestJarSimple option mentioned in minioozie doc does not work (abhishekbafna via rkanter) + OOZIE-2552 Update ActiveMQ version for security and other fixes (asasvari via rkanter) + OOZIE-2571 Add spark.scala.binary.version Maven property so that Scala 2.11 can be used (jonathak via rkanter) + OOZIE-2530 Allow Hive to use a different jline version (poeppt via rkanter) + OOZIE-2660 Create documentation for DB Dump/Load functionality (gezapeti via rkanter) + OOZIE-2659 TestPauseTransitService is flaky (gezapeti via rkanter) + OOZIE-2488 Upgrade to latest OpenJPA version (abhishekbafna via jaydeepvishwakarma) + OOZIE-2648 Child jobs shouldn't send callbacks to Oozie (abhishekbafna via rkanter) + OOZIE-2584 Eliminate Thread.sleep() calls in TestMemoryLocks (pbacsko via rkanter) + OOZIE-2635 TimeZone.getTimeZone has performance issue. (satishsaley via rkanter) + OOZIE-2583 oozie throws EL Exception when reference variable name containing dot (abhishekbafna via jaydeepvishwakarma) + OOZIE-2653 Close active connection to hcat server in fs action (satishsaley via puru) + OOZIE-2617 Read properties files in action configurations (wutaklon via jaydeepvishwakarma) + OOZIE-2615 Flaky tests TestCoordActionsKillXCommand.testActionKillCommandActionNumbers and testActionKillCommandDate (pbacsko via rkanter) + OOZIE-2632 Provide database dump/load functionality to make database migration easier (gezapeti, rjustice via rkanter) + OOZIE-2243 Kill Command does not kill the child job for java action (jaydeepvishwakarma) + OOZIE-2649 Can't override sub-workflow configuration property if defined in parent workflow XML (asasvari via rkanter) + OOZIE-2656 OozieShareLibCLI uses op system username instead of Kerberos to upload jars (gezapeti via rkanter) + OOZIE-1173 Refactor: use ParamChecker inXOozieClient (abhishekbafna via jaydeepvishwakarma) + OOZIE-2657 Clean up redundant access modifiers from oozie interfaces (abhishekbafna via jaydeepvishwakarma) + OOZIE-2517 Add support for startCreatedTime and endCreatedTime filters for coord and bundles (abhishekbafna via jaydeepvishwakarma) + OOZIE-2589 CompletedActionXCommand is hardcoded to wrong priority (tm_linfly via rkanter) + OOZIE-2081 WorkflowJob notification to include coordinator action id (seoeun25 via rkanter) + OOZIE-2036 Drop support for Java 1.6 (gezapeti via jaydeepvishwakarma) + OOZIE-2512 ShareLibservice returns incorrect path for jar (satishsaley via puru) + OOZIE-2508 Documentation change for Coord action rerun [OOZIE-1735] (satishsaley via puru) + OOZIE-2628 Fix Workflow EL function return types in Documentation (abhishekbafna via jaydeepvishwakarma) + OOZIE-2623 Oozie should use a dummy OutputFormat (satishsaley via rohini) + OOZIE-2625 Drop workflowgenerator (rkanter) + OOZIE-2602 Upgrade oozie to pig 0.16.0 (nperiwal via jaydeepvishwakarma) + OOZIE-2493 TestDistcpMain deletes action.xml from wrong filesystem (abhishekbafna via rkanter) + OOZIE-2496 Testcase failure due to environment specific line.separator value (abhishekbafna via rkanter) + OOZIE-2574 Oozie to support replication-enabled mysql urls (abhishekbafna via rkanter) + OOZIE-2433 oozie restart required if oozie metrics to graphing tool broken (nperiwal via jaydeepvishwakarma) + OOZIE-2244 Oozie should mask passwords in the logs when logging command arguments (venkatnrangan via jaydeepvishwakarma) + OOZIE-2516 Update web service documentation for jobs API (abhishekbafna via rkanter) + OOZIE-2497 Some tests fail on windows due to hard coded URIs (abhishekbafna via rkanter) + OOZIE-2349 Method getCoordJobInfo(String jobId, String filter, int offset, int length, boolean desc) is not present in LocalOozieClientCoord (nperiwal via rkanter) + OOZIE-2172 ZooKeeper Security Tests failed with JVM IBM JAVA (GauravPande via rkanter) + OOZIE-2555 Oozie SSL enable setup does not return port for admin -servers (abhishekbafna via rkanter) + OOZIE-2440 Exponential re-try policy for workflow action (satishsaley via jaydeepvishwakarma) + OOZIE-2539 Incorrect property key is used for 'hive log4j configuration file for execution mode' (abhishekbafna via jaydeepvishwakarma) + OOZIE-2565 [Oozie web Console] Make the timezones in settings tab to be sorted by default (meetchandan via jaydeepvishwakarma) + OOZIE-2520 SortBy filter for ordering the jobs query results (abhishekbafna via jaydeepvishwakarma) + OOZIE-2506 Add logs into RecoverService for logging information about queued commands (abhishekbafna via jaydeepvishwakarma) + OOZIE-2515 Duplicate information for "Changing endtime/pausetime of a Bundle Job" in CommandLineTool wiki (abhishekbafna via jaydeepvishwakarma) + OOZIE-2390 Rerun with failed option removing completed output data (jaydeepvishwakarma) + OOZIE-2462 When calling ParamChecker.notNull() in CoordActionsIgnoreXCommand.java, "Action" should be passed instead of "Action cannot be null" (mballur via jaydeepvishwakarma) + OOZIE-2507 Expose monitoring via JMX beans in Oozie (fdenes via rkanter) + OOZIE-2581 Oozie should reset SecurityManager in finally block (satishsaley via rohini) + OOZIE-2579 Bulk kill tests in TestBulkWorkflowXCommand might fail because of a race condition (pbacsko via rkanter) + OOZIE-2587 Disable SchedulerService on certain tests (pbacsko via rkanter) + OOZIE-2603 Give thread pools a meaningful name in CallableQueueService and SchedulerService (pbacsko via rkanter) + OOZIE-2436 Fork/join workflow fails with "oozie.action.yarn.tag must not be null" (puru) + OOZIE-2578 Oozie example distcp job fails to run within an encrypted zone with checksum match error (pbacsko via rkanter) + OOZIE-2362 SQL injection in BulkJPAExecutor (pbacsko via rkanter) + OOZIE-2577 Flaky tests TestCoordActionInputCheckXCommand.testTimeout and testTimeoutWithException (pbacsko via rkanter) + OOZIE-2570 remove -PtestPatchCompile from patch testing as there is no such profile (gezapeti via rkanter) + OOZIE-2504 Create a log4j.properties under HADOOP_CONF_DIR in Shell Action (harsh) + OOZIE-2567 HCat connection is not closed while getting hcat cred (puru) + OOZIE-2547 Add mapreduce.job.cache.files to spark action (satishsaley via rohini) + OOZIE-2550 Flaky tests in TestZKUUIDService.java (pbacsko via rkanter) + OOZIE-2445 Doc for - Specifying coordinator input datasets in more logical ways (puru) + OOZIE-2541 Possible resource leak in Hive2Credentials (pbacsko via rkanter) + OOZIE-2563 Pass spark-defaults.conf to spark action (satishsaley via rohini) + OOZIE-2556 TestAbandonedCoordChecker.testCatchupJob is flaky (puru) + OOZIE-2522 There can be multiple coord submit from bundle in case of ZK glitch (puru) + OOZIE-2553 Cred tag is required for all actions in the workflow even if an action does not require it (me.venkatr via rohini) OOZIE-2503 show ChildJobURLs to spark action (satishsaley via puru) OOZIE-2551 Feature request: epoch timestamp generation (jtolar via puru) OOZIE-2542 Option to disable OpenJPA BrokerImpl finalization (puru) http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/distcp/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/hcatalog/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/hive/pom.xml ---------------------------------------------------------------------- diff --cc sharelib/hive/pom.xml index ba49403,b339b51..f2e4ac0 --- a/sharelib/hive/pom.xml +++ b/sharelib/hive/pom.xml @@@ -142,10 -150,15 +142,15 @@@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-utils</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>jline</groupId> + <artifactId>jline</artifactId> + <version>${hive.jline.version}</version> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/hive2/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java ---------------------------------------------------------------------- diff --cc sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java index 5e71f12,b023b79..72fadcc --- a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java +++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java @@@ -192,7 -191,7 +191,6 @@@ public class TestHive2ActionExecutor ex "<query>" + query + "</query>" + "</hive2>"; } -- @SuppressWarnings("deprecation") public void testHive2Action() throws Exception { setupHiveServer2(); Path inputDir = new Path(getFsTestCaseDir(), INPUT_DIRNAME); @@@ -259,9 -266,51 +251,44 @@@ assertTrue(fs.exists(outputDir)); assertTrue(fs.isDirectory(outputDir)); } + // Negative testcase with incorrect hive-query. + { + String query = getHive2BadScript(inputDir.toString(), outputDir.toString()); + Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME))); + dataWriter.write(SAMPLE_DATA_TEXT); + dataWriter.close(); + Context context = createContext(getQueryActionXml(query)); - final RunningJob launcherJob = submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.2")); - String launcherId = context.getAction().getExternalId(); - waitFor(200 * 1000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return launcherJob.isComplete(); - } - }); - assertTrue(launcherJob.isSuccessful()); ++ final String launcherId = submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.2")); ++ waitUntilYarnAppDoneAndAssertSuccess(launcherId); + Configuration conf = new XConfiguration(); + conf.set("user.name", getTestUser()); + Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), + conf); + assertFalse(LauncherMapperHelper.hasIdSwap(actionData)); + Hive2ActionExecutor ae = new Hive2ActionExecutor(); + ae.check(context, context.getAction()); + assertTrue(launcherId.equals(context.getAction().getExternalId())); + assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); + ae.end(context, context.getAction()); + assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); + assertNull(context.getExternalChildIDs()); + } + } + + private String getHive2BadScript(String inputPath, String outputPath) { + StringBuilder buffer = new StringBuilder(NEW_LINE); + buffer.append("set -v;").append(NEW_LINE); + buffer.append("DROP TABLE IF EXISTS test;").append(NEW_LINE); + buffer.append("CREATE EXTERNAL TABLE test (a INT) STORED AS"); + buffer.append(NEW_LINE).append("TEXTFILE LOCATION '"); + buffer.append(inputPath).append("';").append(NEW_LINE); + buffer.append("INSERT OVERWRITE DIRECTORY '"); + buffer.append(outputPath).append("'").append(NEW_LINE); + buffer.append("SELECT (a-1) FROM test-bad;").append(NEW_LINE); + return buffer.toString(); } - private RunningJob submitAction(Context context, Namespace ns) throws Exception { + private String submitAction(Context context, Namespace ns) throws Exception { Hive2ActionExecutor ae = new Hive2ActionExecutor(); WorkflowAction action = context.getAction(); http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/oozie/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c49f382b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java ---------------------------------------------------------------------- diff --cc sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java index d17c431,30d68e2..0815318 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java @@@ -41,15 -41,14 +41,17 @@@ public class JavaMain extends LauncherM Configuration actionConf = loadActionConf(); setYarnTag(actionConf); + setApplicationTags(actionConf, TEZ_APPLICATION_TAGS); + setApplicationTags(actionConf, SPARK_YARN_TAGS); - LauncherMainHadoopUtils.killChildYarnJobs(actionConf); + LauncherMain.killChildYarnJobs(actionConf); Class<?> klass = actionConf.getClass(JAVA_MAIN_CLASS, Object.class); - System.out.println("Main class : " + klass.getName()); - LauncherMapper.printArgs("Arguments :", args); + System.out.println("Java action main class : " + klass.getName()); + System.out.println("Java action arguments :"); + for (String arg : args) { + System.out.println(" " + arg); + } System.out.println(); Method mainMethod = klass.getMethod("main", String[].class); try {