http://git-wip-us.apache.org/repos/asf/oozie/blob/993f06df/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java b/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java index f90f784..3253512 100644 --- a/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java +++ b/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java @@ -22,16 +22,19 @@ import java.io.File; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; import java.io.StringWriter; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.Properties; import java.util.zip.GZIPOutputStream; import org.apache.oozie.command.CommandException; import org.apache.oozie.service.ServiceException; import org.apache.oozie.service.Services; +import org.apache.oozie.service.XLogService; import org.apache.oozie.test.XTestCase; public class TestLogStreamer extends XTestCase { @@ -39,9 +42,20 @@ public class TestLogStreamer extends XTestCase { static String logStatement = " USER[oozie] GROUP[-] TOKEN[-] APP[-] " + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] "; + String defaultLog4jFile; + private final static SimpleDateFormat filenameDateFormatter = new SimpleDateFormat("yyyy-MM-dd-HH"); + @Override + protected void tearDown() throws Exception { + if (null != defaultLog4jFile) { + setSystemProperty(XLogService.LOG4J_FILE, defaultLog4jFile); + } + super.tearDown(); + } + public void testStreamLog() throws IOException, CommandException, ServiceException, ParseException { + setupLog4j(); new Services().init(); long currTime = System.currentTimeMillis(); @@ -151,7 +165,7 @@ public class TestLogStreamer extends XTestCase { XLogStreamer str = new XLogStreamer(xf, getTestCaseDir(), "oozie.log", 1); Calendar cal = Calendar.getInstance(); cal.set(2009, Calendar.JUNE, 24, 2, 43, 0); - str.streamLog(sw, cal.getTime(), new Date(currTime - 5 * 3600000), 4096); + str.streamLog(sw, cal.getTime(), new Date(currTime - 5 * 3600000)); String[] out = sw.toString().split("\n"); // Check if the retrieved log content is of length seven lines after filtering based on time window, file name // pattern and parameters like JobId, Username etc. and/or based on log level like INFO, DEBUG, etc. @@ -172,7 +186,7 @@ public class TestLogStreamer extends XTestCase { xf.setLogLevel("DEBUG|INFO"); xf.setParameter("JOB", "14-200904160239--example-forkjoinwf"); XLogStreamer str1 = new XLogStreamer(xf, getTestCaseDir(), "oozie.log", 1); - str1.streamLog(sw1, null, null, 4096); + str1.streamLog(sw1, null, null); out = sw1.toString().split("\n"); // Check if the retrieved log content is of length eight lines after filtering based on time window, file name // pattern and parameters like JobId, Username etc. and/or based on log level like INFO, DEBUG, etc. @@ -189,6 +203,7 @@ public class TestLogStreamer extends XTestCase { } public void testStreamLogMultipleHours() throws IOException, CommandException, ServiceException { + setupLog4j(); new Services().init(); long currTime = System.currentTimeMillis(); XLogFilter.reset(); @@ -243,7 +258,7 @@ public class TestLogStreamer extends XTestCase { Calendar calendarEntry = Calendar.getInstance(); // Setting start-time to 2012-04-24-19 for log stream (month-1 passed as parameter since 0=January), and end time is current time calendarEntry.set(2012, 3, 24, 19, 0); - str2.streamLog(sw2, calendarEntry.getTime(), new Date(System.currentTimeMillis()), 4096); + str2.streamLog(sw2, calendarEntry.getTime(), new Date(System.currentTimeMillis())); String[] out = sw2.toString().split("\n"); // Check if the retrieved log content is of length five lines after filtering based on time window, file name @@ -259,6 +274,7 @@ public class TestLogStreamer extends XTestCase { } public void testStreamLogNoDash() throws IOException, CommandException, ServiceException { + setupLog4j(); new Services().init(); long currTime = System.currentTimeMillis(); XLogFilter.reset(); @@ -297,7 +313,7 @@ public class TestLogStreamer extends XTestCase { xf.setLogLevel("DEBUG|INFO"); XLogStreamer str = new XLogStreamer(xf, getTestCaseDir(), "oozie.log", 1); - str.streamLog(sw, null, null, 4096); + str.streamLog(sw, null, null); String[] out = sw.toString().split("\n"); // Check if the retrieved log content is of length five lines after filtering; we expect the first five lines because the // filtering shouldn't care whether or not there is a dash while the last five lines don't pass the normal filtering @@ -310,6 +326,16 @@ public class TestLogStreamer extends XTestCase { assertEquals(true, out[4].contains("_L5_")); } + public void testBufferLen() throws IOException, CommandException, ServiceException { + new Services().init(); + XLogStreamer str = new XLogStreamer(null, getTestCaseDir(), "oozie.log", 1); + assertEquals(4096, str.getBufferLen()); + str = new XLogErrorStreamer(null); + assertEquals(2048, str.getBufferLen()); + str = new XLogAuditStreamer(null); + assertEquals(3, str.getBufferLen()); + } + static void writeToGZFile(File f, StringBuilder sbr) throws IOException { GZIPOutputStream gzout = new GZIPOutputStream(new FileOutputStream(f)); String strg = sbr.toString(); @@ -318,4 +344,17 @@ public class TestLogStreamer extends XTestCase { gzout.write(buf, 0, buf.length); gzout.close(); } + + private void setupLog4j() throws IOException { + defaultLog4jFile = System.getProperty(XLogService.LOG4J_FILE); + File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties"); + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties"); + Properties log4jProps = new Properties(); + log4jProps.load(is); + // prevent conflicts with other tests by changing the log file location + log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log"); + log4jProps.store(new FileOutputStream(log4jFile), ""); + setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName()); + } }
http://git-wip-us.apache.org/repos/asf/oozie/blob/993f06df/core/src/test/java/org/apache/oozie/util/TestSimplifiedTimestampedMessageParser.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestSimplifiedTimestampedMessageParser.java b/core/src/test/java/org/apache/oozie/util/TestSimplifiedTimestampedMessageParser.java index ea899fa..3b2e4b9 100644 --- a/core/src/test/java/org/apache/oozie/util/TestSimplifiedTimestampedMessageParser.java +++ b/core/src/test/java/org/apache/oozie/util/TestSimplifiedTimestampedMessageParser.java @@ -24,11 +24,15 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.StringWriter; + +import org.apache.oozie.service.ServiceException; +import org.apache.oozie.service.Services; import org.apache.oozie.test.XTestCase; public class TestSimplifiedTimestampedMessageParser extends XTestCase { - public void testProcessRemainingLog() throws IOException { + public void testProcessRemainingLog() throws IOException, ServiceException { + new Services().init(); XLogFilter.reset(); XLogFilter.defineParameter("USER"); XLogFilter.defineParameter("GROUP"); @@ -42,7 +46,8 @@ public class TestSimplifiedTimestampedMessageParser extends XTestCase { File file = TestTimestampedMessageParser.prepareFile1(getTestCaseDir()); StringWriter sw = new StringWriter(); - new SimpleTimestampedMessageParser(new BufferedReader(new FileReader(file)), xf).processRemaining(sw, 4096); + new SimpleTimestampedMessageParser(new BufferedReader(new FileReader(file)), xf).processRemaining(sw, + new XLogStreamer(xf)); String[] out = sw.toString().split("\n"); assertEquals(19, out.length); assertTrue(out[0].contains("_L1_")); @@ -66,7 +71,8 @@ public class TestSimplifiedTimestampedMessageParser extends XTestCase { assertTrue(out[18].contains("_L17_")); } - public void testProcessRemainingCoordinatorLogForActions() throws IOException { + public void testProcessRemainingCoordinatorLogForActions() throws IOException, ServiceException { + new Services().init(); XLogFilter.reset(); XLogFilter.defineParameter("USER"); XLogFilter.defineParameter("GROUP"); @@ -80,7 +86,8 @@ public class TestSimplifiedTimestampedMessageParser extends XTestCase { File file = TestTimestampedMessageParser.prepareFile2(getTestCaseDir()); StringWriter sw = new StringWriter(); - new SimpleTimestampedMessageParser(new BufferedReader(new FileReader(file)), xf).processRemaining(sw, 4096); + new SimpleTimestampedMessageParser(new BufferedReader(new FileReader(file)), xf).processRemaining(sw, + new XLogStreamer(xf)); String[] matches = sw.toString().split("\n"); assertEquals(9, matches.length); assertTrue(matches[0].contains("_L1_")); http://git-wip-us.apache.org/repos/asf/oozie/blob/993f06df/core/src/test/java/org/apache/oozie/util/TestTimestampedMessageParser.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestTimestampedMessageParser.java b/core/src/test/java/org/apache/oozie/util/TestTimestampedMessageParser.java index 9e28cbc..859277a 100644 --- a/core/src/test/java/org/apache/oozie/util/TestTimestampedMessageParser.java +++ b/core/src/test/java/org/apache/oozie/util/TestTimestampedMessageParser.java @@ -131,7 +131,8 @@ public class TestTimestampedMessageParser extends XTestCase { try { File file = prepareFile3(getTestCaseDir()); StringWriter sw = new StringWriter(); - new TimestampedMessageParser(new BufferedReader(new FileReader(file)), xf).processRemaining(sw, 4096); + new TimestampedMessageParser(new BufferedReader(new FileReader(file)), xf).processRemaining(sw, + new XLogStreamer(xf)); assertTrue(sw.toString().isEmpty()); } catch (Exception e) { @@ -153,7 +154,8 @@ public class TestTimestampedMessageParser extends XTestCase { xf.setLogLevel("DEBUG|WARN"); File file = prepareFile1(getTestCaseDir()); StringWriter sw = new StringWriter(); - new TimestampedMessageParser(new BufferedReader(new FileReader(file)), xf).processRemaining(sw, 4096); + new TimestampedMessageParser(new BufferedReader(new FileReader(file)), xf).processRemaining(sw, + new XLogStreamer(xf)); String[] out = sw.toString().split("\n"); assertEquals(14, out.length); assertTrue(out[0].contains("_L1_")); @@ -186,7 +188,8 @@ public class TestTimestampedMessageParser extends XTestCase { File file = prepareFile2(getTestCaseDir()); StringWriter sw = new StringWriter(); - new TimestampedMessageParser(new BufferedReader(new FileReader(file)), xf).processRemaining(sw, 4096); + new TimestampedMessageParser(new BufferedReader(new FileReader(file)), xf).processRemaining(sw, + new XLogStreamer(xf)); String[] matches = sw.toString().split("\n"); assertEquals(2, matches.length); assertTrue(matches[0].contains("_L1_")); http://git-wip-us.apache.org/repos/asf/oozie/blob/993f06df/core/src/test/java/org/apache/oozie/util/TestXLogUserFilterParam.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestXLogUserFilterParam.java b/core/src/test/java/org/apache/oozie/util/TestXLogUserFilterParam.java index 46f273f..4ce4651 100644 --- a/core/src/test/java/org/apache/oozie/util/TestXLogUserFilterParam.java +++ b/core/src/test/java/org/apache/oozie/util/TestXLogUserFilterParam.java @@ -180,38 +180,6 @@ public class TestXLogUserFilterParam extends XTestCase { assertTrue(lines[0].contains("E0803: IO error, Variable substitution depth too large: 20 ${dniInputDir}")); } - // Test logduration, out of range - public void testException() throws Exception { - File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties"); - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - InputStream is = cl.getResourceAsStream("test-no-dash-log4j.properties"); - - Properties log4jProps = new Properties(); - log4jProps.load(is); - // prevent conflicts with other tests by changing the log file location - log4jProps.setProperty("log4j.appender.oozie.File", getTestCaseDir() + "/oozie.log"); - log4jProps.store(new FileOutputStream(log4jFile), ""); - setSystemProperty(XLogService.LOG4J_FILE, log4jFile.getName()); - - new Services().init(); - Services.get().getConf().setInt(XLogFilter.MAX_SCAN_DURATION, 10); - Map<String, String[]> paramMap = new HashMap<String, String[]>(); - paramMap.put(RestConstants.LOG_FILTER_OPTION, new String[] {}); - XLogFilter filter = new XLogFilter(new XLogUserFilterParam(paramMap)); - Date startDate = new Date(); - Date endDate = new Date(startDate.getTime() + 60 * 60 * 1000 * 11); - - try { - doStreamLog(filter, startDate, endDate); - fail("should not come here"); - } - catch (Exception e) { - e.printStackTrace(); - assertTrue(e.getMessage().contains( - "Request log streaming time range is higher than configured.")); - } - } - // Test log duration - in range public void testNoException_Withrecent() throws Exception { File log4jFile = new File(getTestCaseConfDir(), "test-log4j.properties"); @@ -397,7 +365,7 @@ public class TestXLogUserFilterParam extends XTestCase { private String doStreamLog(XLogFilter xf, Date startDate, Date endDate) throws Exception { StringWriter w = new StringWriter(); Services.get().get(XLogStreamingService.class) - .streamLog(xf, startDate, endDate, w, new HashMap<String, String[]>()); + .streamLog(new XLogStreamer(xf, null), startDate, endDate, w); return w.toString(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/993f06df/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index ffa5c53..e141d0e 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2815 Oozie not always display job log (puru) OOZIE-2860 Improve Jetty logging (andras.piros via pbacsko) OOZIE-2457 Oozie log parsing regex consume more than 90% cpu (satishsaley) OOZIE-2844 Increase stability of Oozie actions when log4j.properties is missing or not readable (andras.piros via pbacsko) http://git-wip-us.apache.org/repos/asf/oozie/blob/993f06df/webapp/src/main/webapp/oozie-console.js ---------------------------------------------------------------------- diff --git a/webapp/src/main/webapp/oozie-console.js b/webapp/src/main/webapp/oozie-console.js index 76864a9..72c8a19 100644 --- a/webapp/src/main/webapp/oozie-console.js +++ b/webapp/src/main/webapp/oozie-console.js @@ -48,7 +48,7 @@ function getLogs(url, searchFilter, logStatus, textArea, shouldParseResponse, er if (searchFilter) { url = url + "&logfilter=" + searchFilter; } - logStatus.getEl().dom.innerText = "Log status : Loading... done"; + logStatus.getEl().dom.innerText = "Log status : Loading..."; if (!errorMsg) { errorMsg = "Fatal Error. Can't load logs."; @@ -96,7 +96,7 @@ function getLogs(url, searchFilter, logStatus, textArea, shouldParseResponse, er logStatus.getEl().dom.innerText = "Log status : Errored out"; } - if (xhr.readyState =4 && xhr.status == 200) { + if (xhr.readyState == 4 && xhr.status == 200) { logStatus.getEl().dom.innerText = "Log status : Loading... done"; } } catch (e) {
