Repository: oozie Updated Branches: refs/heads/master 9911ebbf4 -> 550850da7
OOZIE-3233 Remove DST shift from the coordinator job's end time (kmarton via andras.piros) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/550850da Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/550850da Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/550850da Branch: refs/heads/master Commit: 550850da7d3cb4ac2e50a90cbdada42d8203f360 Parents: 9911ebb Author: Andras Piros <andras.pi...@cloudera.com> Authored: Thu Jun 7 10:58:52 2018 +0200 Committer: Andras Piros <andras.pi...@cloudera.com> Committed: Thu Jun 7 10:58:52 2018 +0200 ---------------------------------------------------------------------- .../CoordMaterializeTransitionXCommand.java | 12 ------ .../TestCoordMaterializeTransitionXCommand.java | 39 ++++++++++++-------- release-log.txt | 1 + 3 files changed, 24 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/550850da/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java index abea2b7..ec9ef41 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java @@ -437,8 +437,6 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo boolean firstMater = true; - end = new DaylightOffsetCalculator(startMatdTime, endMatdTime).calculate(appTz, end); - while (effStart.compareTo(end) < 0 && (ignoreMaxActions || maxActionToBeCreated-- > 0)) { if (pause != null && effStart.compareTo(pause) >= 0) { break; @@ -573,8 +571,6 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo Calendar end = Calendar.getInstance(); end.setTime(jobEndTime); - end = calculateEndTimeWithDSTOffset(end); - if (end.getTime().compareTo(endMatdTime) <= 0) { LOG.info("[" + job.getId() + "]: all actions have been materialized, set pending to true"); // set doneMaterialization to true when materialization is done @@ -585,14 +581,6 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo job.setNextMaterializedTime(endMatdTime); } - private Calendar calculateEndTimeWithDSTOffset(final Calendar endTime) { - final TimeZone appTz = DateUtils.getTimeZone(coordJob.getTimeZone()); - final Calendar start = Calendar.getInstance(appTz); - start.setTime(startMatdTime); - - return new DaylightOffsetCalculator(startMatdTime, endMatdTime).calculate(appTz, endTime); - } - @Override public String getKey() { return getName() + "_" + jobId; http://git-wip-us.apache.org/repos/asf/oozie/blob/550850da/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java index 76c928c..8189732 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java @@ -404,22 +404,26 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); final String startPlusOneHour = "2013-03-10T09:00Z"; + final String startPlusTwoHours = "2013-03-10T10:00Z"; final Date[] nominalTimesWithDSTChange = new Date[] {DateUtils.parseDateOozieTZ(startInThePast), - DateUtils.parseDateOozieTZ(startPlusOneHour)}; - checkCoordActionsNominalTime(job.getId(), 2, nominalTimesWithDSTChange); + DateUtils.parseDateOozieTZ(startPlusOneHour), + DateUtils.parseDateOozieTZ(startPlusTwoHours) + }; + final int expectedNominalTimeCount = 3; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimesWithDSTChange); - checkTwoActionsAfterCatchup(job); + checkTwoActionsAfterCatchup(job, expectedNominalTimeCount, "2013-03-10T11:00Z"); } - private void checkTwoActionsAfterCatchup(CoordinatorJobBean job) throws ParseException { + private void checkTwoActionsAfterCatchup(CoordinatorJobBean job, int expectedJobCount, String nextMaterialization) + throws ParseException { try { final JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue("coordinator job should have already been materialized", job.isDoneMaterialization()); - assertEquals("coordinator action count mismatch", 2, job.getLastActionNumber()); - final String startPlusThreeHours = "2013-03-10T10:00Z"; + assertEquals("coordinator action count mismatch", expectedJobCount, job.getLastActionNumber()); assertEquals("coordinator next materialization time mismatch", - DateUtils.parseDateOozieTZ(startPlusThreeHours), job.getNextMaterializedTime()); + DateUtils.parseDateOozieTZ(nextMaterialization), job.getNextMaterializedTime()); } catch (final JPAExecutorException se) { se.printStackTrace(); @@ -445,9 +449,10 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { final String startPlusOneHour = "2013-03-10T09:00Z"; final Date[] nominalTimesWithoutDSTChange = new Date[] {DateUtils.parseDateOozieTZ(startInThePast), DateUtils.parseDateOozieTZ(startPlusOneHour)}; - checkCoordActionsNominalTime(job.getId(), 2, nominalTimesWithoutDSTChange); + final int expectedNominalTimeCount = 2; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimesWithoutDSTChange); - checkTwoActionsAfterCatchup(job); + checkTwoActionsAfterCatchup(job, expectedNominalTimeCount, "2013-03-10T10:00Z"); } public void testActionMaterWithDST1() throws Exception { @@ -459,8 +464,9 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-03-10T08:00Z"), DateUtils.parseDateOozieTZ("2013-03-10T09:00Z"), DateUtils.parseDateOozieTZ("2013-03-10T10:00Z"), + DateUtils.parseDateOozieTZ("2013-03-10T11:00Z"), }; - final int expectedNominalTimeCount = 3; + final int expectedNominalTimeCount = 4; checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { @@ -468,7 +474,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue(job.isDoneMaterialization()); assertEquals(expectedNominalTimeCount, job.getLastActionNumber()); - assertEquals(DateUtils.parseDateOozieTZ("2013-03-10T11:00Z"), job.getNextMaterializedTime()); + assertEquals(DateUtils.parseDateOozieTZ("2013-03-10T12:00Z"), job.getNextMaterializedTime()); } catch (JPAExecutorException se) { se.printStackTrace(); @@ -486,9 +492,8 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { DateUtils.parseDateOozieTZ("2012-11-04T08:00Z"), DateUtils.parseDateOozieTZ("2012-11-04T09:00Z"), DateUtils.parseDateOozieTZ("2012-11-04T10:00Z"), - DateUtils.parseDateOozieTZ("2012-11-04T11:00Z"), }; - final int expectedNominalTimeCount = 5; + final int expectedNominalTimeCount = 4; checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { @@ -496,7 +501,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue(job.isDoneMaterialization()); assertEquals(job.getLastActionNumber(), expectedNominalTimeCount); - assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2012-11-04T12:00Z")); + assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2012-11-04T11:00Z")); } catch (JPAExecutorException se) { se.printStackTrace(); @@ -797,7 +802,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { // If the startTime and endTime straddle a DST shift (the Coord is in // "America/Los_Angeles"), then we need to adjust for // that because startTime and endTime assume GMT - next = new Date(startTime.getTime() + TIME_IN_DAY * 3); + next = new Date(startTime.getTime() + TIME_IN_DAY * 4); tz = TimeZone.getTimeZone(job.getTimeZone()); next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next)); assertEquals(next, job.getNextMaterializedTime()); @@ -872,6 +877,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-03-12T01:10")), DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-03-12T03:10")), // DST started DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-03-12T04:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-03-12T05:10")), }; testELAndCronNominalTimesEqual(startTime, endTime, nominalTimesWithOneDstChange, everyHourAtTen, "1", Timeunit.HOUR); @@ -886,7 +892,6 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { DateUtils.parseDateOozieTZ("2017-11-05T08:10Z"), // LA time: 2017-11-05T01:10 DateUtils.parseDateOozieTZ("2017-11-05T09:10Z"), // LA time: 2017-11-05T01:10, DST ended DateUtils.parseDateOozieTZ("2017-11-05T10:10Z"), // LA time: 2017-11-05T02:10 - DateUtils.parseDateOozieTZ("2017-11-05T11:10Z"), // LA time: 2017-11-05T03:10 }; testELAndCronNominalTimesEqual(startTime, endTime, nominalTimesWithOneDstChange, everyHourAtTen, "1", Timeunit.HOUR); @@ -1024,6 +1029,8 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T02:30")), DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T04:00")), DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T04:30")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T05:00")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T05:30")), }; testELAndCronNominalTimesEqual(startTime, endTime, nominalTimesWithTwoDstChange,everyThirtiethMinuteCron, http://git-wip-us.apache.org/repos/asf/oozie/blob/550850da/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index a117bef..3e7e98d 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.1.0 release (trunk - unreleased) +OOZIE-3233 Remove DST shift from the coordinator job's end time (kmarton via andras.piros) OOZIE-1393 Allow sending emails via TLS (mbalakrishnan, dionusos via andras.piros) OOZIE-3156 Retry SSH action check when cannot connect to remote host (txsing via andras.piros) OOZIE-3227 Eliminate duplicate dependencies when using Hadoop 3 DistributedCache (dionusos via andras.piros)
