Repository: oozie Updated Branches: refs/heads/master 35451d505 -> 3c9b299a8
OOZIE-2494 Cron syntax not handling DST properly (kmarton via andras.piros) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3c9b299a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3c9b299a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3c9b299a Branch: refs/heads/master Commit: 3c9b299a87cc500826707a752d4a830f198e1e2a Parents: 35451d5 Author: Andras Piros <andras.pi...@cloudera.com> Authored: Thu May 3 16:29:22 2018 +0200 Committer: Andras Piros <andras.pi...@cloudera.com> Committed: Thu May 3 16:29:22 2018 +0200 ---------------------------------------------------------------------- .../CoordMaterializeTransitionXCommand.java | 29 +- .../TestCoordMaterializeTransitionXCommand.java | 303 ++++++++++++++++++- .../site/twiki/CoordinatorFunctionalSpec.twiki | 18 ++ release-log.txt | 1 + 4 files changed, 338 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/3c9b299a/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java index ff1aa37..92e0dab 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java @@ -472,9 +472,10 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo effStart.add(Calendar.MINUTE, -1); firstMater = false; } - nextTime = CoordCommandUtils.getNextValidActionTimeForCronFrequency(effStart.getTime(), coordJob); + Date prevTime = new Date(effStart.getTimeInMillis()); effStart.setTime(nextTime); + addDSTChangeToNominalTime(prevTime, nextTime, coordJob); } if (effStart.compareTo(end) < 0) { @@ -531,6 +532,31 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo } } + /** + * Apply DST correction according the job`s timezone, if the difference between the previous nominal time and the actual one + * is greater or equal than 24 hours. + * Calendar uses a similar approach: applies DST change if the TimeUnit is lower or equal than TimeUnit.DAY. With this approach + * a similar behaviour can be achieved. + * + * @see {@http://oozie.apache.org/docs/5.0.0/CoordinatorFunctionalSpec.html#a7._Handling_Timezones_and_Daylight_Saving_Time} + * + * @param prevTime nominal time of the previous coordinator action + * @param nextTime nominal time of the actual coordinator action + * @param coordJob the coordinator job + */ + private void addDSTChangeToNominalTime(Date prevTime, Date nextTime, CoordinatorJobBean coordJob) { + final long differenceBetweenTwoActionsInSec = java.util.concurrent.TimeUnit.MILLISECONDS.toSeconds + (Math.abs(prevTime.getTime() - nextTime.getTime())); + final long oneDayInSeconds = java.util.concurrent.TimeUnit.DAYS.toSeconds(1); + if (differenceBetweenTwoActionsInSec < oneDayInSeconds) { + return; + } + final long dstOffset = DaylightOffsetCalculator.getDSTOffset(DateUtils + .getTimeZone(coordJob.getTimeZone()), coordJob.getStartTime(), nextTime); + LOG.debug("[{0}] ms DST offset applied to nominal time for coordinator job: [{1}]", dstOffset, coordJob.getId()); + nextTime.setTime(nextTime.getTime() + dstOffset); + } + private void storeToDB(CoordinatorActionBean actionBean, String actionXml, Configuration jobConf) throws Exception { LOG.debug("In storeToDB() coord action id = " + actionBean.getId() + ", size of actionXml = " + actionXml.length()); @@ -560,7 +586,6 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo // if the job endtime == action endtime, we don't need to materialize this job anymore Date jobEndTime = job.getEndTime(); - if (job.getStatus() == CoordinatorJob.Status.PREP){ LOG.info("[" + job.getId() + "]: Update status from " + job.getStatus() + " to RUNNING"); job.setStatus(Job.Status.RUNNING); http://git-wip-us.apache.org/repos/asf/oozie/blob/3c9b299a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java index 323d34a..76c928c 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java @@ -20,7 +20,9 @@ package org.apache.oozie.command.coord; import java.io.File; import java.sql.Timestamp; +import java.text.DateFormat; import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Calendar; import java.util.Date; @@ -60,6 +62,9 @@ import org.jdom.Element; @SuppressWarnings("deprecation") public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { + private int oneHourInSeconds = (int) java.util.concurrent.TimeUnit.HOURS.toSeconds(1); + + @Override protected void setUp() throws Exception { super.setUp(); @@ -480,7 +485,9 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2012-11-04T07:00Z"), DateUtils.parseDateOozieTZ("2012-11-04T08:00Z"), DateUtils.parseDateOozieTZ("2012-11-04T09:00Z"), - DateUtils.parseDateOozieTZ("2012-11-04T10:00Z"), DateUtils.parseDateOozieTZ("2012-11-04T11:00Z") }; + DateUtils.parseDateOozieTZ("2012-11-04T10:00Z"), + DateUtils.parseDateOozieTZ("2012-11-04T11:00Z"), + }; final int expectedNominalTimeCount = 5; checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); @@ -557,7 +564,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2009-03-06T10:14Z"); Date pauseTime = null; CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, - pauseTime, 300, "5"); + pauseTime, 300, "5", Timeunit.MINUTE); new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); checkCoordActionsTimeout(job.getId() + "@1", 300); } @@ -651,7 +658,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { * * @throws Exception */ - public void testMaterizationLookup() throws Exception { + public void testMaterializationLookup() throws Exception { long TIME_IN_MIN = 60 * 1000; long TIME_IN_HOURS = TIME_IN_MIN * 60; long TIME_IN_DAY = TIME_IN_HOURS * 24; @@ -811,6 +818,276 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { return standard; } + public void testWhenChangingDSTCronAndELMonthlyFrequenciesEqual() throws Exception { + String dstAwareMonthlyCron = "10 23 1 1-12 *"; + Date startTime = DateUtils.parseDateOozieTZ("2016-03-01T23:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2016-12-03T00:00Z");; + Date[] nominalTimesWithTwoDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-01T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-04-01T15:10")), // DST started + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-05-01T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-06-01T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-07-01T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-08-01T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-09-01T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-10-01T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-01T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-12-01T15:10")), // DST ended + }; + testELAndCronNominalTimesEqual(startTime, endTime, nominalTimesWithTwoDstChange, dstAwareMonthlyCron, "1", Timeunit.MONTH); + } + + public void testWhenChangingDSTCronAndELDailyFrequenciesEqual() throws Exception { + String dstAwareDailyCron = "10 23 * * *"; + Date startTime = DateUtils.parseDateOozieTZ("2016-03-11T23:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2016-03-15T02:00Z"); + Date[] nominalTimesWithTwoDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-11T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-12T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T15:10")), // DST started + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-14T15:10")), + }; + testELAndCronNominalTimesEqual(startTime, endTime, nominalTimesWithTwoDstChange, dstAwareDailyCron, "1", Timeunit.DAY); + } + + public void testWhenChangingDSTELEveryTwentyFourthHour() throws Exception { + Date startTime = DateUtils.parseDateOozieTZ("2016-03-11T23:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2016-03-15T02:00Z"); + Date[] nominalTimesWithTwoDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-11T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-12T15:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T16:10")), // DST started + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-14T16:10")), + }; + testELNominalTimes(startTime, endTime, nominalTimesWithTwoDstChange,"24", Timeunit.HOUR); + } + + public void testWhenBeginningDSTCronAndELHourlyFrequenciesEqual() throws Exception { + Date startTime = DateUtils.parseDateOozieTZ("2017-03-12T07:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2017-03-12T12:30Z"); + String everyHourAtTen = "10 * * * *"; + Date[] nominalTimesWithOneDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-03-11T23:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-03-12T00:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-03-12T01:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-03-12T03:10")), // DST started + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-03-12T04:10")), + }; + + testELAndCronNominalTimesEqual(startTime, endTime, nominalTimesWithOneDstChange, everyHourAtTen, "1", Timeunit.HOUR); + } + + public void testWhenEndingDSTCronAndELHourlyFrequenciesEqual() throws Exception { + Date startTime = DateUtils.parseDateOozieTZ("2017-11-05T07:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2017-11-05T10:30Z"); + String everyHourAtTen = "10 * * * *"; + Date[] nominalTimesWithOneDstChange = new Date[]{ + DateUtils.parseDateOozieTZ("2017-11-05T07:10Z"), // LA time: 2017-11-05T00:10 + DateUtils.parseDateOozieTZ("2017-11-05T08:10Z"), // LA time: 2017-11-05T01:10 + DateUtils.parseDateOozieTZ("2017-11-05T09:10Z"), // LA time: 2017-11-05T01:10, DST ended + DateUtils.parseDateOozieTZ("2017-11-05T10:10Z"), // LA time: 2017-11-05T02:10 + DateUtils.parseDateOozieTZ("2017-11-05T11:10Z"), // LA time: 2017-11-05T03:10 + }; + + testELAndCronNominalTimesEqual(startTime, endTime, nominalTimesWithOneDstChange, everyHourAtTen, "1", Timeunit.HOUR); + } + + public void testWhenChangingDSTELEveryTwentiethDay() throws Exception { + Date startTime = DateUtils.parseDateOozieTZ("2016-02-01T13:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2016-12-03T00:00Z"); + Date[] nominalTimesWithTwoDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-02-01T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-02-21T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-12T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-04-01T05:10")), // DST started + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-04-21T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-05-11T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-05-31T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-06-20T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-07-10T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-07-30T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-08-19T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-09-08T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-09-28T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-10-18T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-07T05:10")), // DST ended + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-27T05:10")), + }; + + testELNominalTimes(startTime, endTime, nominalTimesWithTwoDstChange,"20", Timeunit.DAY); + } + + public void testWhenChangingDSTCronEveryTwentiethDay() throws Exception { + Date startTime = DateUtils.parseDateOozieTZ("2016-02-01T13:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2016-12-03T00:00Z"); + String everyTwentiethDayAroundDstShift = "10 13 */20 2-3,11,12 *"; + + Date[] nominalTimesWithTwoDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-02-01T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-02-21T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-01T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-21T05:10")), // DST started + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-01T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-21T05:10")), // DST ended + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-12-01T05:10")), + }; + testCronNominalTimes(startTime, endTime, nominalTimesWithTwoDstChange, everyTwentiethDayAroundDstShift); + } + + public void testWhenChangingDSTCronAndELEveryThirdMonthFrequenciesEqual() throws Exception { + Date startTime = DateUtils.parseDateOozieTZ("2016-01-01T13:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2017-12-03T00:00Z"); + String everyThirdMonth = "10 13 1 */3 *"; + + Date[] nominalTimesWithTwoDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-01-01T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-04-01T05:10")), // DST started on 13th March + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-07-01T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-10-01T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-01-01T05:10")), // DST ended on 6th of November + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-04-01T05:10")), // DST started again on 12th March + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-07-01T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2017-10-01T05:10")), + }; + testELAndCronNominalTimesEqual(startTime, endTime, nominalTimesWithTwoDstChange, everyThirdMonth, "3", Timeunit.MONTH); + } + + public void testWhenDSTStartsCronFrequencyEveryTwentiethHour() throws Exception { + Date startTime = DateUtils.parseDateOozieTZ("2016-01-01T13:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2016-12-03T00:00Z"); + String everyTwentiethHourNearDSTShift = "10 */20 12-14 3 *"; + Date[] nominalTimesWithTwoDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-11T16:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-12T12:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-12T16:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T13:10")), // DST change + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T17:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-14T13:10")), + }; + testCronNominalTimes(startTime, endTime, nominalTimesWithTwoDstChange, everyTwentiethHourNearDSTShift); + } + + public void testWhenDSTStartsELFrequencyEveryTwentiethHour() throws Exception { + Date startTime = DateUtils.parseDateOozieTZ("2016-03-12T13:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2016-03-16T00:00Z"); + Date[] nominalTimesWithTwoDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-12T05:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T01:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T22:10")), // DST started + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-14T18:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-15T14:10")), + }; + + testELNominalTimes(startTime, endTime, nominalTimesWithTwoDstChange, "20", Timeunit.HOUR); + } + + public void testWhenDSTSEndsCronFrequencyEveryTwentiethHour() throws Exception { + Date startTime = DateUtils.parseDateOozieTZ("2016-01-01T13:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2016-12-03T00:00Z"); + String everyTwentiethHourNearDSTShift = "10 */20 5-7 11 *"; + Date[] nominalTimesWithTwoDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-04T16:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-05T13:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-05T17:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-06T12:10")), // DST change + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-06T16:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-07T12:10")), + }; + + testCronNominalTimes(startTime, endTime, nominalTimesWithTwoDstChange, everyTwentiethHourNearDSTShift); + } + + public void testWhenDSTEndsELFrequencyEveryTwentiethHour() throws Exception { + Date startTime = DateUtils.parseDateOozieTZ("2016-11-04T23:10Z"); + Date endTime = DateUtils.parseDateOozieTZ("2016-11-08T22:00Z"); + Date[] nominalTimesWithTwoDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-04T16:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-05T12:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-06T07:10")), // DST ended + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-07T03:10")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-11-07T23:10")), + }; + + testELNominalTimes(startTime, endTime, nominalTimesWithTwoDstChange, "20", Timeunit.HOUR); + } + + public void testWhenDSTSwitchELAndCronFrequencyEveryThirtiethMinute() throws Exception { + Date startTime = DateUtils.parseDateOozieTZ("2016-03-13T08:00Z"); + Date endTime = DateUtils.parseDateOozieTZ("2016-03-13T13:00Z"); + String everyThirtiethMinuteCron = "*/30 * * * *"; + Date[] nominalTimesWithTwoDstChange = new Date[]{ + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T00:00")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T00:30")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T01:00")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T01:30")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T02:00")), // DST change + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T02:30")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T04:00")), + DateUtils.parseDateOozieTZ(convertLATimeToUTC("2016-03-13T04:30")), + }; + + testELAndCronNominalTimesEqual(startTime, endTime, nominalTimesWithTwoDstChange,everyThirtiethMinuteCron, + "30", Timeunit.MINUTE); + } + + private String convertLATimeToUTC (String localTime) throws Exception { + DateFormat LATimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"); + LATimeFormat.setTimeZone(TimeZone.getTimeZone("America/Los_Angeles")); + Date date = LATimeFormat.parse(localTime); + + DateFormat utcFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); + utcFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + + return utcFormat.format(date); + + } + + private void testELAndCronNominalTimesEqual (Date startTime, Date endTime, Date[] nominalTimes, String cronFrequency, + String elFrequency, Timeunit elTimeUnit) throws Exception { + CoordinatorJobBean elJob = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, + elFrequency, elTimeUnit); + CoordinatorJobBean cronJob = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, + cronFrequency); + + new CoordMaterializeTransitionXCommand(elJob.getId(), oneHourInSeconds).call(); + new CoordMaterializeTransitionXCommand(cronJob.getId(), oneHourInSeconds).call(); + + + JPAService jpaService = Services.get().get(JPAService.class); + + elJob = jpaService.execute(new CoordJobGetJPAExecutor(elJob.getId())); + cronJob = jpaService.execute(new CoordJobGetJPAExecutor(cronJob.getId())); + + checkCoordActionsNominalTime(cronJob.getId(), nominalTimes.length, nominalTimes); + checkCoordActionsNominalTime(elJob.getId(), nominalTimes.length, nominalTimes); + + assertTrue("Cron and EL job materialization should both be complete", + elJob.isDoneMaterialization() && cronJob.isDoneMaterialization()); + } + + private void testCronNominalTimes (Date startTime, Date endTime, Date[] nominalTimes, String cronFrequency) throws Exception { + CoordinatorJobBean cronJob = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, + cronFrequency); + new CoordMaterializeTransitionXCommand(cronJob.getId(), oneHourInSeconds).call(); + + JPAService jpaService = Services.get().get(JPAService.class); + cronJob = jpaService.execute(new CoordJobGetJPAExecutor(cronJob.getId())); + checkCoordActionsNominalTime(cronJob.getId(), nominalTimes.length, nominalTimes); + assertTrue("Cron job materialization should be complete", cronJob.isDoneMaterialization()); + } + + private void testELNominalTimes (Date startTime, Date endTime, Date[] nominalTimes, String elFrequency, Timeunit elTimeUnit) + throws Exception { + CoordinatorJobBean elJob = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, + elFrequency, elTimeUnit); + new CoordMaterializeTransitionXCommand(elJob.getId(), oneHourInSeconds).call(); + + JPAService jpaService = Services.get().get(JPAService.class); + elJob = jpaService.execute(new CoordJobGetJPAExecutor(elJob.getId())); + checkCoordActionsNominalTime(elJob.getId(), nominalTimes.length, nominalTimes); + assertTrue("EL job materialization should be complete", elJob.isDoneMaterialization()); + } + public void testLastOnlyMaterialization() throws Exception { long now = System.currentTimeMillis(); @@ -968,35 +1245,39 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, Date startTime, Date endTime, Date pauseTime, String freq) throws Exception { - return addRecordToCoordJobTable(status, startTime, endTime, pauseTime, -1, freq); + return addRecordToCoordJobTable(status, startTime, endTime, pauseTime, -1, freq, Timeunit.MINUTE); + } + protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, Date startTime, Date endTime, + Date pauseTime, String freq, Timeunit timeUnit) throws Exception { + return addRecordToCoordJobTable(status, startTime, endTime, pauseTime, -1, freq, timeUnit); } protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, Date startTime, Date endTime, Date pauseTime, String freq, int matThrottling) throws Exception { - return addRecordToCoordJobTable(status, startTime, endTime, pauseTime, -1, freq, CoordinatorJob.Execution.FIFO, - matThrottling); + return addRecordToCoordJobTable(status, startTime, endTime, pauseTime, -1, freq, Timeunit.MINUTE, + CoordinatorJob.Execution.FIFO, matThrottling); } protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, Date startTime, Date endTime, - Date pauseTime, int timeout, String freq) throws Exception { - return addRecordToCoordJobTable(status, startTime, endTime, pauseTime, timeout, freq, + Date pauseTime, int timeout, String freq, Timeunit timeUnit) throws Exception { + return addRecordToCoordJobTable(status, startTime, endTime, pauseTime, timeout, freq, timeUnit, CoordinatorJob.Execution.FIFO, 20); } protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, Date startTime, Date endTime, Date pauseTime, int timeout, String freq, CoordinatorJob.Execution execution) throws Exception { - return addRecordToCoordJobTable(status, startTime, endTime, pauseTime, timeout, freq, execution, 20); + return addRecordToCoordJobTable(status, startTime, endTime, pauseTime, timeout, freq, Timeunit.MINUTE, execution, 20); } protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, Date startTime, Date endTime, - Date pauseTime, int timeout, String freq, CoordinatorJob.Execution execution, int matThrottling) + Date pauseTime, int timeout, String freq, Timeunit timeUnit, CoordinatorJob.Execution execution, int matThrottling) throws Exception { CoordinatorJobBean coordJob = createCoordJob(status, startTime, endTime, false, false, 0); coordJob.setStartTime(startTime); coordJob.setEndTime(endTime); coordJob.setPauseTime(pauseTime); coordJob.setFrequency(freq); - coordJob.setTimeUnit(Timeunit.MINUTE); + coordJob.setTimeUnit(timeUnit); coordJob.setTimeout(timeout); coordJob.setConcurrency(3); coordJob.setMatThrottling(matThrottling); http://git-wip-us.apache.org/repos/asf/oozie/blob/3c9b299a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki index 69e1b33..bc32806 100644 --- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki +++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki @@ -3661,6 +3661,24 @@ Each nested operation can be named and passed into the workflow using coord:data As mentioned in section #4.1.1 'Timezones and Daylight-Saving', the coordinator engine works exclusively in UTC, and dataset and application definitions are always expressed in UTC. + +*%GREEN% Example of nominal times in case of DST change: %ENDCOLOR%* + +| *Frequency* | *Timezone* | *Nominal times in local time* | *Comments* | +| =${coord:months(1)}= or =${10 23 1 1-12 *}= | America/Los_Angeles | 2016-03-01T15:10 <br/> 2016-04-01T15:10 <br/> 2016-05-01T15:10 <br/> ... <br/> 2016-11-01T15:10 <br/> 2016-12-01T15:10 | <br/>DST Start on March 13, 2:00 am <br/><br/><br/><br/> DST End on November 6, 2:00 am| +| =${coord:month(3)} or =${10 13 1 */3 *}= | America/Los_Angeles |2016-01-01T05:10 <br/> 2016-04-01T05:10 <br/> 2016-07-01T05:10 <br/> 2016-10-01T05:10 <br/> 2017-01-01T05:10 <br/> 2017-04-01T05:10 <br/> 2017-07-01T05:10 | <br/> DST Start on 2016 March 13, 2:00 am <br/><br/><br/>DST End on 2016 November 6, 2:00 am <br/> DST Start on 2017 March 12, 2:00 am| +| =${coord:days(20)}=| America/Los_Angeles | 2016-03-12T05:10 <br/> 2016-04-01T05:10 <br/> 2016-04-21T05:10 <br/> ... <br/> 2016-11-07T05:10 <br/> 2016-11-27T05:10 |<br/> DST Start on March 13, 2:00 am <br/><br/><br/> DST End on November 6, 2:00 am| +| =${10 13 */20 * *}= | America/Los_Angeles | 2016-03-01T05:10 <br/> 2016-03-21T05:10 <br/> 2016-11-01T05:10 <br/> 2016-11-21T05:10 <br/> 2016-12-01T05:10 | <br/> DST Start on March 13, 2:00 am <br/><br/> DST End on November 6, 2:00| +| =${coord:days(1)}= or =${10 23 * * *}= | America/Los_Angeles | 2016-03-11T15:10 <br/> 2016-03-12T15:10 <br/> 2016-03-13T15:10 <br/> 2016-03-14T15:10 | <br/> DST Start on March 13, 2:00 am| +| =${coord:hours(24)}=| America/Los_Angeles | 2016-03-11T15:10 <br/> 2016-03-12T15:10 <br/> 2016-03-13T16:10 <br/> 2016-03-14T16:10 | <br/><br/> DST Start on March 13, 2:00 am, but since the time unit is in hours, there will be a shift in local time| +| =${coord:hours(1)}= or =${10 * * * *}= | America/Los_Angeles | 2017-03-12T00:10 <br/> 2017-03-12T01:10 <br/> 2017-03-12T03:10 <br/> 2017-03-12T04:10 | <br/><br/> DST Start on March 12, 2:00 am, so hour 2 will be skipped| +| =${coord:hours(1)}= or =${10 * * * *}= | America/Los_Angeles | 2017-11-05T00:10 <br/> 2017-11-05T01:10 <br/> 2017-11-05T01:10 <br/> 2017-11-05T02:10 <br/> 2017-11-05T03:10 | <br/><br/> DST End on November 5, 2:00 am, so hour 1 will be doubled| +| =${10 */20 12-14 3 *}= | America/Los_Angeles | 2016-03-12T12:10 <br/> 2016-03-12T16:10 <br/> 2016-03-13T13:10 <br/> 2016-03-13T17:10 <br/> 2016-03-14T13:10 <br/> ... <br/> 2016-11-05T17:10 <br/> 2016-11-06T12:10 <br/> 2016-11-06T16:10 | <br/> <br/> DST Start on March 13, 2:00 am, so after this time the nominal times will be shifted <br/><br/><br/> <br/> DST End on November 6, 2:00 am| +| =${coord:hours(20)}= | America/Los_Angeles |2016-03-12T05:10 <br/> 2016-03-13T01:10 <br/> 2016-03-13T22:10 <br/> 2016-03-14T18:10 <br/> 2016-03-15T14:10 <br/> ... <br/> 2016-11-05T12:10 <br/> 2016-11-06T07:10 <br/> 2016-11-07T03:10 <br/> 2016-11-07T23:10 | <br/><br/> DST Start on March 13, 2:00, so here will be 21 hours in local time between the two materialization times <br/><br/><br/><br/><br/> DST End on November 6, 2:00 am, so here will be a 19 hour difference in local time| +| =${coord:minutes(30)}= or =${*/30 * * * *}= | America/Los_Angeles | 2016-03-13T01:00 <br/> 2016-03-13T01:30 <br/> 2016-03-13T02:00 <br/> 2016-03-13T02:30 <br/> 2016-03-13T04:00 <br/> 2016-03-13T04:30 | <br/><br/><br/> DST Start on March 13, 2:00 am| + +*IMPORTANT:* Please note, that in the actual implementation, DST corrections are not applied in case of higher frequencies than one day, so for this frequencies, some shifting in nominal times are expected. + ---+++ 7.1. Handling Timezones with No Day Light Saving Time For timezones that don't observe day light saving time, handling timezones offsets is trivial. http://git-wip-us.apache.org/repos/asf/oozie/blob/3c9b299a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 4e026f5..8508f39 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.1.0 release (trunk - unreleased) +OOZIE-2494 Cron syntax not handling DST properly (kmarton via andras.piros) OOZIE-2427 [Kerberos] Authentication failure for the javascript resources under /ext-2.2 (lianggz via andras.piros) OOZIE-3221 Rename DEFAULT_LAUNCHER_MAX_ATTEMPS (dbist13 via andras.piros) OOZIE-3222 The description for DAG is not accurate in the documentation (gongchuanjie via gezapeti)
