HIVE-10795: Remove use of PerfLogger from Orc (Owen O'Malley reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/527497cc Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/527497cc Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/527497cc Branch: refs/heads/beeline-cli Commit: 527497cc85144013ca0bb44fb3aa1fba9de8e052 Parents: 1280cca Author: Prasanth Jayachandran <j.prasant...@gmail.com> Authored: Tue Jul 7 23:08:07 2015 -0700 Committer: Prasanth Jayachandran <j.prasant...@gmail.com> Committed: Tue Jul 7 23:08:07 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 20 ++++++++++---------- .../hive/ql/io/orc/OrcNewInputFormat.java | 16 ++++++++++------ .../apache/hadoop/hive/ql/log/PerfLogger.java | 1 - 3 files changed, 20 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/527497cc/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 7346bc4..8864013 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -58,7 +58,6 @@ import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; -import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -122,9 +121,6 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, private static final long DEFAULT_MIN_SPLIT_SIZE = 16 * 1024 * 1024; private static final long DEFAULT_MAX_SPLIT_SIZE = 256 * 1024 * 1024; - private static final PerfLogger perfLogger = PerfLogger.getPerfLogger(); - private static final String CLASS_NAME = ReaderImpl.class.getName(); - /** * When picking the hosts for a split that crosses block boundaries, * any drop any host that has fewer than MIN_INCLUDED_LOCATION of the @@ -490,7 +486,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, context.numFilesCounter.incrementAndGet(); FileInfo fileInfo = Context.footerCache.getIfPresent(file.getPath()); if (fileInfo != null) { - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Info cached for path: " + file.getPath()); } if (fileInfo.modificationTime == file.getModificationTime() && @@ -501,7 +497,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, } else { // Invalidate Context.footerCache.invalidate(file.getPath()); - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Meta-Info for : " + file.getPath() + " changed. CachedModificationTime: " + fileInfo.modificationTime + ", CurrentModificationTime: " @@ -511,7 +507,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, } } } else { - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Info not cached for path: " + file.getPath()); } } @@ -871,7 +867,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, includeStripe[i] = (i >= stripeStats.size()) || isStripeSatisfyPredicate(stripeStats.get(i), sarg, filterColumns); - if (LOG.isDebugEnabled() && !includeStripe[i]) { + if (isDebugEnabled && !includeStripe[i]) { LOG.debug("Eliminating ORC stripe-" + i + " of file '" + file.getPath() + "' as it did not satisfy " + "predicate condition."); @@ -1060,9 +1056,13 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, @Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ORC_GET_SPLITS); + if (isDebugEnabled) { + LOG.debug("getSplits started"); + } List<OrcSplit> result = generateSplitsInfo(job, numSplits); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ORC_GET_SPLITS); + if (isDebugEnabled) { + LOG.debug("getSplits finished"); + } return result.toArray(new InputSplit[result.size()]); } http://git-wip-us.apache.org/repos/asf/hive/blob/527497cc/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java index b6ad0dc..1833d3d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java @@ -21,10 +21,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputFormat; @@ -37,8 +37,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit; /** An InputFormat for ORC files. Keys are meaningless, * value is the OrcStruct object */ public class OrcNewInputFormat extends InputFormat<NullWritable, OrcStruct>{ - private static final PerfLogger perfLogger = PerfLogger.getPerfLogger(); - private static final String CLASS_NAME = ReaderImpl.class.getName(); + + private static final Log LOG = LogFactory.getLog(OrcNewInputFormat.class); @Override public RecordReader<NullWritable, OrcStruct> createRecordReader( @@ -117,7 +117,9 @@ public class OrcNewInputFormat extends InputFormat<NullWritable, OrcStruct>{ @Override public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ORC_GET_SPLITS); + if (LOG.isDebugEnabled()) { + LOG.debug("getSplits started"); + } List<OrcSplit> splits = OrcInputFormat.generateSplitsInfo(ShimLoader.getHadoopShims() .getConfiguration(jobContext)); @@ -125,7 +127,9 @@ public class OrcNewInputFormat extends InputFormat<NullWritable, OrcStruct>{ for(OrcSplit split: splits) { result.add(new OrcNewSplit(split)); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ORC_GET_SPLITS); + if (LOG.isDebugEnabled()) { + LOG.debug("getSplits finished"); + } return result; } http://git-wip-us.apache.org/repos/asf/hive/blob/527497cc/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index f202991..20ca195 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -63,7 +63,6 @@ public class PerfLogger { public static final String TEZ_RUN_PROCESSOR = "TezRunProcessor"; public static final String TEZ_INIT_OPERATORS = "TezInitializeOperators"; public static final String LOAD_HASHTABLE = "LoadHashtable"; - public static final String ORC_GET_SPLITS = "OrcGetSplits"; public static final String SPARK_SUBMIT_TO_RUNNING = "SparkSubmitToRunning"; public static final String SPARK_BUILD_PLAN = "SparkBuildPlan";