Author: szehon
Date: Sat Jul 12 22:17:30 2014
New Revision: 1610072
URL: http://svn.apache.org/r1610072
Log:
HIVE-7291: Refactor TestParser to understand test-property file (Szehon,
reviewed by Brock)
Modified:
hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java
Modified:
hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
URL:
http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java?rev=1610072&r1=1610071&r2=1610072&view=diff
==============================================================================
---
hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
(original)
+++
hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
Sat Jul 12 22:17:30 2014
@@ -21,12 +21,24 @@ package org.apache.hive.ptest.execution.
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Properties;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.PatternLayout;
import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
@@ -97,6 +109,8 @@ public class TestParser {
return result;
}
private List<QFileTestBatch> parseQFileTests() {
+ Map<String, Properties> properties = parseQTestProperties();
+
Splitter splitter = Splitter.on(" ").trimResults().omitEmptyStrings();
List<QFileTestBatch> result = Lists.newArrayList();
for(String alias : context.getString("qFileTests", "").split(" ")) {
@@ -111,22 +125,25 @@ public class TestParser {
for(String excludedTestGroup :
splitter.split(testContext.getString("exclude", ""))) {
excludedTests.addAll(Arrays.asList(testContext.
getString(Joiner.on(".").join("groups", excludedTestGroup),
"").trim().split(" ")));
+ expandTestProperties(excludedTests, properties);
}
Set<String> isolatedTests = Sets.newHashSet();
for(String ioslatedTestGroup :
splitter.split(testContext.getString("isolate", ""))) {
isolatedTests.addAll(Arrays.asList(testContext.
getString(Joiner.on(".").join("groups", ioslatedTestGroup),
"").trim().split(" ")));
+ expandTestProperties(isolatedTests, properties);
}
Set<String> includedTests = Sets.newHashSet();
for(String includedTestGroup :
splitter.split(testContext.getString("include", ""))) {
includedTests.addAll(Arrays.asList(testContext.
getString(Joiner.on(".").join("groups", includedTestGroup),
"").trim().split(" ")));
+ expandTestProperties(includedTests, properties);
}
- if(!includedTests.isEmpty() && !excludedTests.isEmpty()) {
- throw new IllegalArgumentException(String.format("Included and
excluded mutally exclusive." +
- " Included = %s, excluded = %s", includedTests.toString(),
excludedTests.toString()));
- }
+
+ //excluded overrides included
+ includedTests.removeAll(excludedTests);
+
result.addAll(createQFileTestBatches(
driver,
checkNotNull(testContext.getString("queryFilesProperty"),
"queryFilesProperty").trim(),
@@ -175,6 +192,72 @@ public class TestParser {
return testBatches;
}
+ /**
+ * @return properties loaded from files specified in
qFileTests.propertyFiles.${fileName}=${filePath}
+ */
+ private Map<String, Properties> parseQTestProperties() {
+ Map<String, String> propFiles =
context.getSubProperties("qFileTests.propertyFiles.");
+ Map<String, Properties> propertyMap = new HashMap<String, Properties>();
+ for (String propFile : propFiles.keySet()) {
+ Properties properties = new Properties();
+ String path = sourceDirectory + File.separator + propFiles.get(propFile);
+ FileInputStream fis = null;
+ try {
+ fis = new FileInputStream(path);
+ properties.load(fis);
+ } catch (IOException e) {
+ logger.warn("Error processing Qtest property file", e);
+ throw new IllegalArgumentException("Error processing Qtest property
file: " + path);
+ } finally {
+ try {
+ if (fis != null) {
+ fis.close();
+ }
+ } catch (IOException e) { //ignore
+ }
+ }
+ propertyMap.put(propFile, properties);
+ logger.info("Loaded Qtest property file: " + path);
+ }
+ return propertyMap;
+ }
+
+ /**
+ * If any of given tests are of the form: ${fileName}.${property} (test list
within a property file),
+ * then expand them. Then remove those markers from the list of tests.
+ */
+ private void expandTestProperties(Set<String> tests, Map<String, Properties>
propMap) {
+ Set<String> toRemove = new HashSet<String>();
+ Set<String> toAdd = new HashSet<String>();
+
+ String pattern = "([^\\.]*)\\.\\$\\{([^}]*)}";
+ Pattern r = Pattern.compile(pattern);
+ for (String test : tests) {
+ Matcher m = r.matcher(test);
+ if (m.find()) {
+ toRemove.add(test);
+ logger.info("Expanding qfile property: " + test);
+ String propName = m.group(1);
+ String propValue = m.group(2);
+ Properties props = propMap.get(propName);
+ if (props == null) {
+ logger.warn("No properties found for : " + propName);
+ throw new IllegalArgumentException("No properties found for : " +
propName);
+ }
+ String result = (String) props.get(propValue);
+ if (result == null || result.isEmpty()) {
+ logger.warn("No properties found in file: " + propName + " for
property: " + propValue);
+ throw new IllegalArgumentException("No propertifies found in file: "
+ propName + " for property: " + propValue);
+ }
+ Iterable<String> splits =
Splitter.on(',').trimResults().omitEmptyStrings().split(result);
+ for (String split : splits) {
+ toAdd.add(split);
+ }
+ }
+ }
+ tests.removeAll(toRemove);
+ tests.addAll(toAdd);
+ }
public Supplier<List<TestBatch>> parse() {
return new Supplier<List<TestBatch>>() {
@@ -184,4 +267,25 @@ public class TestParser {
}
};
}
+
+ /**
+ * Manually test this against any property file.
+ * @param args
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length < 1) {
+ throw new IllegalArgumentException("Enter the property file location");
+ }
+ Logger log = LoggerFactory
+ .getLogger(TestParser.class);
+ File workingDir = new File("../..");
+ File testConfigurationFile = new File(args[0]);
+ TestConfiguration conf = TestConfiguration.fromFile(testConfigurationFile,
log);
+ TestParser testParser = new TestParser(conf.getContext(), "test",
workingDir, log);
+ List<TestBatch> testBatches = testParser.parse().get();
+ for (TestBatch testBatch : testBatches) {
+ System.out.println(testBatch.getTestArguments());
+ }
+ }
}
Modified:
hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java
URL:
http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java?rev=1610072&r1=1610071&r2=1610072&view=diff
==============================================================================
---
hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java
(original)
+++
hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java
Sat Jul 12 22:17:30 2014
@@ -19,7 +19,10 @@
package org.apache.hive.ptest.execution.conf;
import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
import java.util.List;
+import java.util.Properties;
import junit.framework.Assert;
@@ -46,6 +49,7 @@ public class TestTestParser {
private File unitTestDir1;
private File unitTestDir2;
private File qFileTestDir;
+ private File propertyDir;
@Before
public void setup() throws Exception {
@@ -56,6 +60,7 @@ public class TestTestParser {
"test", "classes")));
unitTestDir2 = Dirs.create(new File(baseDir, Joiner.on("/").join("source",
"build", "2", "units", "test", "classes")));
qFileTestDir = Dirs.create(new File(baseDir, Joiner.on("/").join("source",
"qfiles")));
+ propertyDir = Dirs.create(new File(baseDir, Joiner.on("/").join("source",
"props")));
Assert.assertTrue((new File(unitTestDir1, "TestA.class")).createNewFile());
Assert.assertTrue((new File(unitTestDir2, "TestB.class")).createNewFile());
Assert.assertTrue((new File(unitTestDir1, "TestC.class")).createNewFile());
@@ -64,10 +69,20 @@ public class TestTestParser {
Assert.assertTrue((new File(qFileTestDir, ".svn")).mkdirs());
Assert.assertTrue((new File(qFileTestDir, "dir.q")).mkdirs());
Assert.assertTrue((new File(qFileTestDir, "normal.q")).createNewFile());
+ Assert.assertTrue((new File(qFileTestDir, "normal2.q")).createNewFile());
+ Assert.assertTrue((new File(qFileTestDir, "normal3.q")).createNewFile());
+ Assert.assertTrue((new File(qFileTestDir, "normal4.q")).createNewFile());
Assert.assertTrue((new File(qFileTestDir, "excluded.q")).createNewFile());
Assert.assertTrue((new File(qFileTestDir, "isolated.q")).createNewFile());
Assert.assertTrue((new File(qFileTestDir, "included.q")).createNewFile());
+ Properties normalProp = new Properties();
+ normalProp.setProperty("normal.one.group", "normal.q,normal2.q");
+ normalProp.setProperty("normal.two.group", "normal3.q,normal4.q");
+ normalProp.setProperty("excluded.group", "excluded.q");
+ normalProp.setProperty("isolated.group", "isolated.q");
+ normalProp.setProperty("included.group", "included.q");
+ serialize("normal.properties", normalProp);
}
@After
public void teardown() {
@@ -107,4 +122,35 @@ public class TestTestParser {
List<TestBatch> testBatches = testParser.parse().get();
Assert.assertEquals(4, testBatches.size());
}
+ @Test
+ public void testParsePropertyFile() throws Exception {
+ context.put("unitTests.directories", "build/1 build/2");
+ context.put("unitTests.include", "TestA TestB");
+ context.put("unitTests.isolate", "TestB");
+ context.put("qFileTests", "f");
+ context.put("qFileTests.propertyFiles.prop",
+ "props" + File.separator + "normal.properties");
+ context.put("qFileTest.f.driver", DRIVER);
+ context.put("qFileTest.f.directory", "qfiles");
+ context.put("qFileTest.f.include", "included");
+ context.put("qFileTest.f.isolate", "isolated");
+ context.put("qFileTest.f.exclude", "excluded");
+ context.put("qFileTest.f.queryFilesProperty", "qfile");
+ context.put("qFileTest.f.groups.included", "prop.${normal.one.group}
prop.${normal.two.group} prop.${isolated.group}");
+ context.put("qFileTest.f.groups.isolated", "prop.${isolated.group}");
+ context.put("qFileTest.f.groups.excluded", "prop.${excluded.group}");
+ testParser = new TestParser(context, "testcase", workingDirectory, LOG);
+ List<TestBatch> testBatches = testParser.parse().get();
+ Assert.assertEquals(4, testBatches.size());
+ }
+
+ private void serialize(String propFileName, Properties props) throws
Exception {
+ File f = new File(propertyDir, propFileName);
+ OutputStream out = new FileOutputStream(f);
+ try {
+ props.store(out, null);
+ } finally {
+ out.close();
+ }
+ }
}
