Repository: hive Updated Branches: refs/heads/master dd5121c7b -> c737dbd8f
HIVE-20360: QTest: ignore driver/qtest exclusions if -Dqfile param is set (Laszlo Bodor via ) Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c737dbd8 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c737dbd8 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c737dbd8 Branch: refs/heads/master Commit: c737dbd8f1db679e20c04b2e58808ce3dbbd0710 Parents: dd5121c7 Author: Laszlo Bodor <bodorlaszlo0...@gmail.com> Authored: Fri Nov 16 13:06:24 2018 +0100 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Fri Nov 16 13:06:24 2018 +0100 ---------------------------------------------------------------------- .../hive/cli/control/AbstractCliConfig.java | 17 ++++++-- .../apache/hadoop/hive/ql/QOutProcessor.java | 45 +++++++++++++++++--- .../org/apache/hadoop/hive/ql/QTestUtil.java | 35 +++++---------- .../hadoop/hive/ql/TestQOutProcessor.java | 2 +- 4 files changed, 62 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c737dbd8/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java index 7151372..142f903 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java @@ -33,6 +33,7 @@ import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.FsType; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.apache.hive.testutils.HiveTestEnvSetup; @@ -214,7 +215,7 @@ public abstract class AbstractCliConfig { // dedup file list Set<File> testFiles = new TreeSet<>(); - if (queryFile != null && !queryFile.equals("")) { + if (isQFileSpecified()) { // The user may have passed a list of files - comma separated for (String qFile : TEST_SPLITTER.split(queryFile)) { File qF; @@ -223,10 +224,10 @@ public abstract class AbstractCliConfig { } else { qF = new File(qFile); } - if (excludedQueryFileNames.contains(qFile)) { + if (excludedQueryFileNames.contains(qFile) && !isQFileSpecified()) { LOG.warn(qF.getAbsolutePath() + " is among the excluded query files for this driver." + " Please update CliConfigs.java or testconfiguration.properties file to" - + " include the qfile"); + + " include the qfile or specify qfile through command line explicitly: -Dqfile=test.q"); } testFiles.add(qF); } @@ -241,12 +242,20 @@ public abstract class AbstractCliConfig { } for (String qFileName : excludedQueryFileNames) { - testFiles.remove(new File(queryDir, qFileName)); + // in case of running as ptest, exclusions should be respected, + // because test drivers receive every qfiles regardless of exclusions + if ("hiveptest".equals(System.getProperty("user.name")) || !isQFileSpecified()) { + testFiles.remove(new File(queryDir, qFileName)); + } } return testFiles; } + public boolean isQFileSpecified() { + return queryFile != null && !queryFile.equals(""); + } + private void prepareDirs() throws Exception { File hiveRootDir = new File(HIVE_ROOT); if (!hiveRootDir.exists()) { http://git-wip-us.apache.org/repos/asf/hive/blob/c737dbd8/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java index a17c508..254cc95 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java @@ -25,7 +25,9 @@ import java.io.FileOutputStream; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -52,6 +54,10 @@ public class QOutProcessor { public static final String MASK_PATTERN = "#### A masked pattern was here ####"; public static final String PARTIAL_MASK_PATTERN = "#### A PARTIAL masked pattern was here ####"; + private final Set<String> qMaskStatsQuerySet = new HashSet<String>(); + private final Set<String> qMaskDataSizeQuerySet = new HashSet<String>(); + private final Set<String> qMaskLineageQuerySet = new HashSet<String>(); + private static final PatternReplacementPair MASK_STATS = new PatternReplacementPair( Pattern.compile(" Num rows: [1-9][0-9]* Data size: [1-9][0-9]*"), " Num rows: ###Masked### Data size: ###Masked###"); @@ -62,6 +68,10 @@ public class QOutProcessor { Pattern.compile("POSTHOOK: Lineage: .*"), "POSTHOOK: Lineage: ###Masked###"); + private static final Pattern PATTERN_MASK_STATS = Pattern.compile("-- MASK_STATS"); + private static final Pattern PATTERN_MASK_DATA_SIZE = Pattern.compile("-- MASK_DATA_SIZE"); + private static final Pattern PATTERN_MASK_LINEAGE = Pattern.compile("-- MASK_LINEAGE"); + private FsType fsType = FsType.local; public static class LineProcessingResult { @@ -141,7 +151,8 @@ public class QOutProcessor { return patterns; } - public void maskPatterns(String fname, boolean maskStats, boolean maskDataSize, boolean maskLineage) throws Exception { + public void maskPatterns(String fname, String tname) throws Exception { + String line; BufferedReader in; BufferedWriter out; @@ -156,7 +167,7 @@ public class QOutProcessor { boolean lastWasMasked = false; while (null != (line = in.readLine())) { - LineProcessingResult result = processLine(line, maskStats, maskDataSize, maskLineage); + LineProcessingResult result = processLine(line, tname); if (result.line.equals(MASK_PATTERN)) { // We're folding multiple masked lines into one. @@ -178,7 +189,7 @@ public class QOutProcessor { out.close(); } - public LineProcessingResult processLine(String line, boolean maskStats, boolean maskDataSize, boolean maskLineage) { + public LineProcessingResult processLine(String line, String tname) { LineProcessingResult result = new LineProcessingResult(line); Matcher matcher = null; @@ -216,7 +227,7 @@ public class QOutProcessor { } } - if (!result.partialMaskWasMatched && maskStats) { + if (!result.partialMaskWasMatched && qMaskStatsQuerySet.contains(tname)) { matcher = MASK_STATS.pattern.matcher(result.line); if (matcher.find()) { result.line = result.line.replaceAll(MASK_STATS.pattern.pattern(), MASK_STATS.replacement); @@ -224,7 +235,7 @@ public class QOutProcessor { } } - if (!result.partialMaskWasMatched && maskDataSize) { + if (!result.partialMaskWasMatched && qMaskDataSizeQuerySet.contains(tname)) { matcher = MASK_DATA_SIZE.pattern.matcher(result.line); if (matcher.find()) { result.line = result.line.replaceAll(MASK_DATA_SIZE.pattern.pattern(), MASK_DATA_SIZE.replacement); @@ -232,7 +243,7 @@ public class QOutProcessor { } } - if (!result.partialMaskWasMatched && maskLineage) { + if (!result.partialMaskWasMatched && qMaskLineageQuerySet.contains(tname)) { matcher = MASK_LINEAGE.pattern.matcher(result.line); if (matcher.find()) { result.line = result.line.replaceAll(MASK_LINEAGE.pattern.pattern(), MASK_LINEAGE.replacement); @@ -298,5 +309,25 @@ public class QOutProcessor { public void addPatternWithMaskComment(String patternStr, String maskComment) { patternsWithMaskComments.add(toPatternPair(patternStr, maskComment)); } - + + public void initMasks(File qf, String query) { + if (matches(PATTERN_MASK_STATS, query)) { + qMaskStatsQuerySet.add(qf.getName()); + } + if (matches(PATTERN_MASK_DATA_SIZE, query)) { + qMaskDataSizeQuerySet.add(qf.getName()); + } + if (matches(PATTERN_MASK_LINEAGE, query)) { + qMaskLineageQuerySet.add(qf.getName()); + } + } + + private boolean matches(Pattern pattern, String query) { + Matcher matcher = pattern.matcher(query); + if (matcher.find()) { + return true; + } + return false; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/c737dbd8/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 59c7ac4..dcf2d49 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -156,6 +156,7 @@ public class QTestUtil { private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically target private static final String TEST_SRC_TABLES_PROPERTY = "test.src.tables"; + /** * The default Erasure Coding Policy to use in Erasure Coding tests. */ @@ -173,9 +174,6 @@ public class QTestUtil { private final Set<String> qHashQuerySet; private final Set<String> qSortNHashQuerySet; private final Set<String> qNoSessionReuseQuerySet; - private final Set<String> qMaskStatsQuerySet; - private final Set<String> qMaskDataSizeQuerySet; - private final Set<String> qMaskLineageQuerySet; private static final String SORT_SUFFIX = ".sorted"; private static Set<String> srcTables; private final Set<String> srcUDFs; @@ -495,9 +493,6 @@ public class QTestUtil { qHashQuerySet = new HashSet<String>(); qSortNHashQuerySet = new HashSet<String>(); qNoSessionReuseQuerySet = new HashSet<String>(); - qMaskStatsQuerySet = new HashSet<String>(); - qMaskDataSizeQuerySet = new HashSet<String>(); - qMaskLineageQuerySet = new HashSet<String>(); this.clusterType = testArgs.getClusterType(); HadoopShims shims = ShimLoader.getHadoopShims(); @@ -534,12 +529,16 @@ public class QTestUtil { this.initScript = scriptsDir + File.separator + testArgs.getInitScript(); this.cleanupScript = scriptsDir + File.separator + testArgs.getCleanupScript(); - overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite")); + overWrite = shouldOverwriteResults(); init(); savedConf = new HiveConf(conf); } + private boolean shouldOverwriteResults() { + return "true".equalsIgnoreCase(System.getProperty("test.output.overwrite")); + } + private String getScriptsDir() { // Use the current directory if it is not specified String scriptsDir = conf.get("test.data.scripts"); @@ -802,15 +801,7 @@ public class QTestUtil { qNoSessionReuseQuerySet.add(qf.getName()); } - if (matches(MASK_STATS, query)) { - qMaskStatsQuerySet.add(qf.getName()); - } - if (matches(MASK_DATA_SIZE, query)) { - qMaskDataSizeQuerySet.add(qf.getName()); - } - if (matches(MASK_LINEAGE, query)) { - qMaskLineageQuerySet.add(qf.getName()); - } + qOutProcessor.initMasks(qf, query); } private static final Pattern SORT_BEFORE_DIFF = Pattern.compile("-- SORT_BEFORE_DIFF"); @@ -818,9 +809,6 @@ public class QTestUtil { private static final Pattern HASH_QUERY_RESULTS = Pattern.compile("-- HASH_QUERY_RESULTS"); private static final Pattern SORT_AND_HASH_QUERY_RESULTS = Pattern.compile("-- SORT_AND_HASH_QUERY_RESULTS"); private static final Pattern NO_SESSION_REUSE = Pattern.compile("-- NO_SESSION_REUSE"); - private static final Pattern MASK_STATS = Pattern.compile("-- MASK_STATS"); - private static final Pattern MASK_DATA_SIZE = Pattern.compile("-- MASK_DATA_SIZE"); - private static final Pattern MASK_LINEAGE = Pattern.compile("-- MASK_LINEAGE"); private boolean matches(Pattern pattern, String query) { Matcher matcher = pattern.matcher(query); @@ -1532,8 +1520,7 @@ public class QTestUtil { File f = new File(logDir, tname + outFileExtension); - qOutProcessor.maskPatterns(f.getPath(), - qMaskStatsQuerySet.contains(tname), qMaskDataSizeQuerySet.contains(tname), qMaskLineageQuerySet.contains(tname)); + qOutProcessor.maskPatterns(f.getPath(), tname); QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(), outFileName, false, qSortSet.contains(tname)); @@ -1549,11 +1536,9 @@ public class QTestUtil { public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs) throws Exception { assert outputs.size() > 1; - qOutProcessor.maskPatterns(outputs.get(0), - qMaskStatsQuerySet.contains(tname), qMaskDataSizeQuerySet.contains(tname), qMaskLineageQuerySet.contains(tname)); + qOutProcessor.maskPatterns(outputs.get(0), tname); for (int i = 1; i < outputs.size(); ++i) { - qOutProcessor.maskPatterns(outputs.get(i), - qMaskStatsQuerySet.contains(tname), qMaskDataSizeQuerySet.contains(tname), qMaskLineageQuerySet.contains(tname)); + qOutProcessor.maskPatterns(outputs.get(i), tname); QTestProcessExecResult result = executeDiffCommand( outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname)); if (result.getReturnCode() != 0) { http://git-wip-us.apache.org/repos/asf/hive/blob/c737dbd8/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java ---------------------------------------------------------------------- diff --git a/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java b/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java index c193da3..9e19f27 100644 --- a/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java +++ b/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java @@ -68,6 +68,6 @@ public class TestQOutProcessor { } private String processLine(String line) { - return qOutProcessor.processLine(line, false, false, false).get(); + return qOutProcessor.processLine(line, null).get(); } } \ No newline at end of file
