This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 772bfda HIVE-23323: Add qsplits profile (Zoltan Haindrich reviewed by Miklos Gergely) 772bfda is described below commit 772bfdadab6f3d8c3a54431fe7b6e8b115e99b29 Author: Zoltan Haindrich <k...@rxd.hu> AuthorDate: Tue May 5 15:35:31 2020 +0000 HIVE-23323: Add qsplits profile (Zoltan Haindrich reviewed by Miklos Gergely) Signed-off-by: Zoltan Haindrich <zhaindr...@cloudera.com> --- itests/bin/generate-cli-splits.sh | 26 ++++++++ itests/qtest-spark/pom.xml | 50 +++++++++++++++- .../hive/cli/TestMiniSparkOnYarnCliDriver.java | 7 ++- .../apache/hadoop/hive/cli/TestSparkCliDriver.java | 7 ++- itests/qtest/pom.xml | 51 +++++++++++++++- .../org/apache/hadoop/hive/cli/TestCliDriver.java | 5 +- .../hive/cli/TestEncryptedHDFSCliDriver.java | 5 +- .../hadoop/hive/cli/TestMiniLlapCliDriver.java | 5 +- .../hive/cli/TestMiniLlapLocalCliDriver.java | 5 +- .../hadoop/hive/cli/control/SplitSupport.java | 69 ++++++++++++++++++++++ .../hadoop/hive/cli/control/TestSplitSupport.java | 48 +++++++++++++++ .../control/splitsupport/SplitSupportDummy.java | 5 ++ .../splitsupport/split0/SplitSupportDummy.java | 5 ++ .../splitsupport/split125/SplitSupportDummy.java | 5 ++ 14 files changed, 281 insertions(+), 12 deletions(-) diff --git a/itests/bin/generate-cli-splits.sh b/itests/bin/generate-cli-splits.sh new file mode 100755 index 0000000..c16d369 --- /dev/null +++ b/itests/bin/generate-cli-splits.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +usage() { + echo "$0 <from> <to>" + exit 1 +} + +[ "$1" == "" ] && usage +[ "$2" == "" ] && usage + + +inDir="$1" +outDir="$2" + +git grep SplitSupport.process | grep "$1" | cut -d ':' -f1 | while read f;do + + echo "processing: $f" + n="`grep N_SPLITS "$f" | cut -d= -f2 | tr -c -d '0-9'`" + echo " * nSplits: $n" + + for((i=0;i<n;i++)) { + oDir="`dirname $f | sed "s|$inDir|$outDir|"`/split$i" + mkdir -p $oDir + cat $f | sed -r "s|^(package.*);$|\1.split$i;|g" > $oDir/`basename $f` + } +done diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml index 60d032d..4f97e29 100644 --- a/itests/qtest-spark/pom.xml +++ b/itests/qtest-spark/pom.xml @@ -417,5 +417,53 @@ </plugin> </plugins> </build> - + <profiles> + <profile> + <id>qsplits</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>generate-split-tests</id> + <phase>generate-sources</phase> + <configuration> + <target> + <exec executable="bash" failonerror="true"> + <arg value="${basedir}/../bin/generate-cli-splits.sh"/> + <arg value="src/test/java"/> + <arg value="target/generated-test-sources"/> + </exec> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>add-test-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-test-source</goal> + </goals> + <configuration> + <sources> + <source>target/generated-test-sources</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> diff --git a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java index 889d0f2..c19d4db 100644 --- a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java +++ b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - + package org.apache.hadoop.hive.cli; import java.io.File; @@ -23,6 +23,7 @@ import java.util.List; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -34,11 +35,13 @@ import org.junit.runners.Parameterized.Parameters; @RunWith(Parameterized.class) public class TestMiniSparkOnYarnCliDriver { + private static final int N_SPLITS = 5; + static CliAdapter adapter = new CliConfigs.SparkOnYarnCliConfig().getCliAdapter(); @Parameters(name = "{0}") public static List<Object[]> getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestMiniSparkOnYarnCliDriver.class, N_SPLITS); } @ClassRule diff --git a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java index def73d4..bf200b6 100644 --- a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java +++ b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - + package org.apache.hadoop.hive.cli; import java.io.File; @@ -23,6 +23,7 @@ import java.util.List; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -34,11 +35,13 @@ import org.junit.runners.Parameterized.Parameters; @RunWith(Parameterized.class) public class TestSparkCliDriver { + private static final int N_SPLITS = 11; + static CliAdapter adapter = new CliConfigs.SparkCliConfig().getCliAdapter(); @Parameters(name = "{0}") public static List<Object[]> getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestSparkCliDriver.class, N_SPLITS); } @ClassRule diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml index 1cc562b..d9492fa 100644 --- a/itests/qtest/pom.xml +++ b/itests/qtest/pom.xml @@ -434,9 +434,55 @@ <test.warehouse.scheme></test.warehouse.scheme> </properties> </profile> + <profile> + <id>qsplits</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>generate-split-tests</id> + <phase>generate-sources</phase> + <configuration> + <target> + <exec executable="bash" failonerror="true"> + <arg value="${basedir}/../bin/generate-cli-splits.sh"/> + <arg value="src/test/java"/> + <arg value="target/generated-test-sources"/> + </exec> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>add-test-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-test-source</goal> + </goals> + <configuration> + <sources> + <source>target/generated-test-sources</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> </profiles> - - <build> + <build> <plugins> <plugin> <groupId>org.codehaus.mojo</groupId> @@ -539,5 +585,4 @@ </plugin> </plugins> </build> - </project> diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java index 4617d31..ccc8a66 100644 --- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java +++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestCliDriver.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -33,11 +34,13 @@ import org.junit.runners.Parameterized.Parameters; @RunWith(Parameterized.class) public class TestCliDriver { + private static final int N_SPLITS = 17; + static CliAdapter adapter = new CliConfigs.CliConfig().getCliAdapter(); @Parameters(name = "{0}") public static List<Object[]> getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestCliDriver.class, N_SPLITS); } @ClassRule diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java index 76fce63..8b6efdb 100644 --- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java +++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestEncryptedHDFSCliDriver.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -33,11 +34,13 @@ import org.junit.runners.Parameterized.Parameters; @RunWith(Parameterized.class) public class TestEncryptedHDFSCliDriver { + private static final int N_SPLITS = 3; + static CliAdapter adapter = new CliConfigs.EncryptedHDFSCliConfig().getCliAdapter(); @Parameters(name = "{0}") public static List<Object[]> getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestEncryptedHDFSCliDriver.class, N_SPLITS); } @ClassRule diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java index cc05c11..c0dd2bf 100644 --- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java +++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapCliDriver.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -33,11 +34,13 @@ import org.junit.runners.Parameterized.Parameters; @RunWith(Parameterized.class) public class TestMiniLlapCliDriver { + private static final int N_SPLITS = 3; + static CliAdapter adapter = new CliConfigs.MiniLlapCliConfig().getCliAdapter(); @Parameters(name = "{0}") public static List<Object[]> getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestMiniLlapCliDriver.class, N_SPLITS); } @ClassRule diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java index 66d7bda..06b802b 100644 --- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java +++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapLocalCliDriver.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.hadoop.hive.cli.control.CliAdapter; import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.apache.hadoop.hive.cli.control.SplitSupport; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -35,9 +36,11 @@ public class TestMiniLlapLocalCliDriver { static CliAdapter adapter = new CliConfigs.MiniLlapLocalCliConfig().getCliAdapter(); + private static int N_SPLITS = 32; + @Parameters(name = "{0}") public static List<Object[]> getParameters() throws Exception { - return adapter.getParameters(); + return SplitSupport.process(adapter.getParameters(), TestMiniLlapLocalCliDriver.class, N_SPLITS); } @ClassRule diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/SplitSupport.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/SplitSupport.java new file mode 100644 index 0000000..a51c5e4 --- /dev/null +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/SplitSupport.java @@ -0,0 +1,69 @@ +package org.apache.hadoop.hive.cli.control; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.common.annotations.VisibleForTesting; + +public class SplitSupport { + + public static List<Object[]> process(List<Object[]> parameters, Class<?> currentClass, int nSplits) { + if (!isSplitExecution(currentClass)) { + return parameters; + } + // auto-disable primary test in case splits are present + if (isSplit0ClassExistsFor(currentClass)) { + return new ArrayList<>(); + } + int i = getSplitIndex(currentClass); + return getSplitParams(parameters, i, nSplits); + } + + private static boolean isSplitExecution(Class<?> currentClass) { + return isSplitClass(currentClass) || isSplit0ClassExistsFor(currentClass); + } + + @VisibleForTesting + static List<Object[]> getSplitParams(List<Object[]> parameters, int i, int nSplits) { + if(i<0 || i>=nSplits) { + throw new IllegalArgumentException("unexpected"); + } + int n = parameters.size(); + int st = i * n / nSplits; + int ed = (i + 1) * n / nSplits; + + return parameters.subList(st, ed); + } + + @VisibleForTesting + static boolean isSplitClass(Class<?> currentClass) { + Package p = currentClass.getPackage(); + return p.getName().matches(".*split[0-9]+$"); + } + + @VisibleForTesting + static int getSplitIndex(Class<?> currentClass) { + Package p = currentClass.getPackage(); + Pattern pat = Pattern.compile("(.*split)([0-9]+)$"); + Matcher matcher = pat.matcher(p.getName()); + if (matcher.find()) { + return Integer.parseInt(matcher.group(2)); + } + throw new IllegalArgumentException("cant get splitindex for: " + p); + } + + @VisibleForTesting + static boolean isSplit0ClassExistsFor(Class<?> clazz) { + Package p = clazz.getPackage(); + String split1 = p.getName() + ".split0." + clazz.getSimpleName(); + try { + Class<?> c = Class.forName(split1); + return c != null; + } catch (ClassNotFoundException e) { + return false; + } + } + +} diff --git a/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/TestSplitSupport.java b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/TestSplitSupport.java new file mode 100644 index 0000000..2e5eede --- /dev/null +++ b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/TestSplitSupport.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.cli.control; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class TestSplitSupport { + + @Test + public void testIsSplitClass1() { + Class<?> mainClass = org.apache.hadoop.hive.cli.control.splitsupport.SplitSupportDummy.class; + Class<?> split0Class = + org.apache.hadoop.hive.cli.control.splitsupport.split0.SplitSupportDummy.class; + assertFalse(SplitSupport.isSplitClass(mainClass)); + assertTrue(SplitSupport.isSplitClass(split0Class)); + } + + @Test + public void testGetSplitIndex0() { + Class<?> split0Class = org.apache.hadoop.hive.cli.control.splitsupport.split0.SplitSupportDummy.class; + assertEquals(0, SplitSupport.getSplitIndex(split0Class)); + } + + @Test + public void testGetSplitIndex125() { + Class<?> split0Class = org.apache.hadoop.hive.cli.control.splitsupport.split125.SplitSupportDummy.class; + assertEquals(125, SplitSupport.getSplitIndex(split0Class)); + } +} diff --git a/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/SplitSupportDummy.java b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/SplitSupportDummy.java new file mode 100644 index 0000000..335a50e --- /dev/null +++ b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/SplitSupportDummy.java @@ -0,0 +1,5 @@ +package org.apache.hadoop.hive.cli.control.splitsupport; + +public class SplitSupportDummy { + +} diff --git a/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split0/SplitSupportDummy.java b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split0/SplitSupportDummy.java new file mode 100644 index 0000000..4ea134d --- /dev/null +++ b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split0/SplitSupportDummy.java @@ -0,0 +1,5 @@ +package org.apache.hadoop.hive.cli.control.splitsupport.split0; + +public class SplitSupportDummy { + +} diff --git a/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split125/SplitSupportDummy.java b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split125/SplitSupportDummy.java new file mode 100644 index 0000000..254f8aa --- /dev/null +++ b/itests/util/src/test/java/org/apache/hadoop/hive/cli/control/splitsupport/split125/SplitSupportDummy.java @@ -0,0 +1,5 @@ +package org.apache.hadoop.hive.cli.control.splitsupport.split125; + +public class SplitSupportDummy { + +}