RANGER-1501: Audit Flush to HDFS does not actually cause the audit logs to be flushed to HDFS - improvement patch
Signed-off-by: rmani <rm...@hortonworks.com> (cherry picked from commit 1befffcc1c18679cdf2c6413496b2078f6d2a838) Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/a3be2390 Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/a3be2390 Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/a3be2390 Branch: refs/heads/ranger-0.7 Commit: a3be2390eeae26fa41147270d358e51a552fa55c Parents: 302c20a Author: Vishal Suvagia <vishalsuva...@apache.org> Authored: Wed Sep 26 19:41:08 2018 +0530 Committer: Pradeep <prad...@apache.org> Committed: Wed Sep 26 20:55:33 2018 +0530 ---------------------------------------------------------------------- .../audit/destination/HDFSAuditDestination.java | 26 ++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/a3be2390/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java index 889b6ff..8b17fc5 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java @@ -63,6 +63,7 @@ public class HDFSAuditDestination extends AuditDestination { private String logFolder; private PrintWriter logWriter = null; + volatile FSDataOutputStream ostream = null; // output stream wrapped in logWriter private String currentFileName; @@ -169,6 +170,7 @@ public class HDFSAuditDestination extends AuditDestination { addDeferredCount(events.size()); out.close(); logWriter = null; + ostream = null; return false; } } catch (Throwable t) { @@ -187,10 +189,22 @@ public class HDFSAuditDestination extends AuditDestination { @Override public void flush() { - if ( logWriter != null) { - logWriter.flush(); - logger.info("Flush HDFS audit logs completed....."); - } + logger.info("Flush called. name=" + getName()); + if (ostream != null) { + try { + synchronized (this) { + if (ostream != null) + // 1) PrinterWriter does not have bufferring of its own so + // we need to flush its underlying stream + // 2) HDFS flush() does not really flush all the way to disk. + ostream.hflush(); + logger.info("Flush HDFS audit logs completed....."); + } + } catch (IOException e) { + logger.error("Error on flushing log writer: " + e.getMessage() + + "\nException will be ignored. name=" + getName() + ", fileName=" + currentFileName); + } + } } /* @@ -246,6 +260,7 @@ public class HDFSAuditDestination extends AuditDestination { + getName() + ", fileName=" + currentFileName); } logWriter = null; + ostream = null; } logStatus(); } @@ -290,7 +305,7 @@ public class HDFSAuditDestination extends AuditDestination { // Create the file to write logger.info("Creating new log file. hdfPath=" + fullPath); - FSDataOutputStream ostream = fileSystem.create(hdfPath); + ostream = fileSystem.create(hdfPath); logWriter = new PrintWriter(ostream); currentFileName = fullPath; } @@ -341,6 +356,7 @@ public class HDFSAuditDestination extends AuditDestination { } logWriter = null; + ostream = null; currentFileName = null; if (!rollOverByDuration) {
