hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sze...@apache.org
Subject svn commit: r1610072 - in /hive/trunk/testutils/ptest2/src: main/java/org/apache/hive/ptest/execution/conf/TestParser.java test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java
Date Sat, 12 Jul 2014 22:17:30 GMT
Author: szehon
Date: Sat Jul 12 22:17:30 2014
New Revision: 1610072

URL: http://svn.apache.org/r1610072
Log:
HIVE-7291: Refactor TestParser to understand test-property file (Szehon, reviewed by Brock)

Modified:
    hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
    hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java

Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java?rev=1610072&r1=1610071&r2=1610072&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
(original)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
Sat Jul 12 22:17:30 2014
@@ -21,12 +21,24 @@ package org.apache.hive.ptest.execution.
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.PatternLayout;
 import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
@@ -97,6 +109,8 @@ public class TestParser {
     return result;
   }
   private List<QFileTestBatch> parseQFileTests() {
+    Map<String, Properties> properties = parseQTestProperties();
+
     Splitter splitter = Splitter.on(" ").trimResults().omitEmptyStrings();
     List<QFileTestBatch> result = Lists.newArrayList();
     for(String alias : context.getString("qFileTests", "").split(" ")) {
@@ -111,22 +125,25 @@ public class TestParser {
       for(String excludedTestGroup : splitter.split(testContext.getString("exclude", "")))
{
         excludedTests.addAll(Arrays.asList(testContext.
             getString(Joiner.on(".").join("groups", excludedTestGroup), "").trim().split("
")));
+        expandTestProperties(excludedTests, properties);
       }
       Set<String> isolatedTests = Sets.newHashSet();
       for(String ioslatedTestGroup : splitter.split(testContext.getString("isolate", "")))
{
         isolatedTests.addAll(Arrays.asList(testContext.
             getString(Joiner.on(".").join("groups", ioslatedTestGroup), "").trim().split("
")));
+        expandTestProperties(isolatedTests, properties);
       }
 
       Set<String> includedTests = Sets.newHashSet();
       for(String includedTestGroup : splitter.split(testContext.getString("include", "")))
{
         includedTests.addAll(Arrays.asList(testContext.
             getString(Joiner.on(".").join("groups", includedTestGroup), "").trim().split("
")));
+        expandTestProperties(includedTests, properties);
       }
-      if(!includedTests.isEmpty() && !excludedTests.isEmpty()) {
-        throw new IllegalArgumentException(String.format("Included and excluded mutally exclusive."
+
-            " Included = %s, excluded = %s", includedTests.toString(), excludedTests.toString()));
-      }
+
+      //excluded overrides included
+      includedTests.removeAll(excludedTests);
+
       result.addAll(createQFileTestBatches(
           driver,
           checkNotNull(testContext.getString("queryFilesProperty"), "queryFilesProperty").trim(),
@@ -175,6 +192,72 @@ public class TestParser {
     return testBatches;
   }
 
+  /**
+   * @return properties loaded from files specified in qFileTests.propertyFiles.${fileName}=${filePath}
+   */
+  private Map<String, Properties> parseQTestProperties() {
+    Map<String, String> propFiles = context.getSubProperties("qFileTests.propertyFiles.");
+    Map<String, Properties> propertyMap = new HashMap<String, Properties>();
+    for (String propFile : propFiles.keySet()) {
+      Properties properties = new Properties();
+      String path = sourceDirectory + File.separator + propFiles.get(propFile);
+      FileInputStream fis = null;
+      try {
+        fis = new FileInputStream(path);
+        properties.load(fis);
+      } catch (IOException e) {
+        logger.warn("Error processing Qtest property file", e);
+        throw new IllegalArgumentException("Error processing Qtest property file: " + path);
+      } finally {
+        try {
+          if (fis != null) {
+            fis.close();
+          }
+        } catch (IOException e) { //ignore
+        }
+      }
+      propertyMap.put(propFile, properties);
+      logger.info("Loaded Qtest property file: " + path);
+    }
+    return propertyMap;
+  }
+
+  /**
+   * If any of given tests are of the form: ${fileName}.${property} (test list within a property
file),
+   * then expand them.  Then remove those markers from the list of tests.
+   */
+  private void expandTestProperties(Set<String> tests, Map<String, Properties>
propMap) {
+    Set<String> toRemove = new HashSet<String>();
+    Set<String> toAdd = new HashSet<String>();
+
+    String pattern = "([^\\.]*)\\.\\$\\{([^}]*)}";
+    Pattern r = Pattern.compile(pattern);
+    for (String test : tests) {
+      Matcher m = r.matcher(test);
+      if (m.find()) {
+        toRemove.add(test);
+        logger.info("Expanding qfile property: " + test);
+        String propName = m.group(1);
+        String propValue = m.group(2);
+        Properties props = propMap.get(propName);
+        if (props == null) {
+          logger.warn("No properties found for : " + propName);
+          throw new IllegalArgumentException("No properties found for : " + propName);
+        }
+        String result = (String) props.get(propValue);
+        if (result == null || result.isEmpty()) {
+          logger.warn("No properties found in file: " + propName + " for property: " + propValue);
+          throw new IllegalArgumentException("No propertifies found in file: " + propName
+ " for property: " + propValue);
+        }
+        Iterable<String> splits = Splitter.on(',').trimResults().omitEmptyStrings().split(result);
+        for (String split : splits) {
+          toAdd.add(split);
+        }
+      }
+    }
+    tests.removeAll(toRemove);
+    tests.addAll(toAdd);
+  }
 
   public Supplier<List<TestBatch>> parse() {
     return new Supplier<List<TestBatch>>() {
@@ -184,4 +267,25 @@ public class TestParser {
       }
     };
   }
+
+  /**
+   * Manually test this against any property file.
+   * @param args
+   * @throws Exception
+   */
+  public static void main(String[] args) throws Exception {
+    if (args.length < 1) {
+      throw new IllegalArgumentException("Enter the property file location");
+    }
+    Logger log = LoggerFactory
+        .getLogger(TestParser.class);
+    File workingDir = new File("../..");
+    File testConfigurationFile = new File(args[0]);
+    TestConfiguration conf = TestConfiguration.fromFile(testConfigurationFile, log);
+    TestParser testParser = new TestParser(conf.getContext(), "test", workingDir, log);
+    List<TestBatch> testBatches = testParser.parse().get();
+    for (TestBatch testBatch : testBatches) {
+      System.out.println(testBatch.getTestArguments());
+    }
+  }
 }

Modified: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java?rev=1610072&r1=1610071&r2=1610072&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java
(original)
+++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestTestParser.java
Sat Jul 12 22:17:30 2014
@@ -19,7 +19,10 @@
 package org.apache.hive.ptest.execution.conf;
 
 import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
 import java.util.List;
+import java.util.Properties;
 
 import junit.framework.Assert;
 
@@ -46,6 +49,7 @@ public class TestTestParser {
   private File unitTestDir1;
   private File unitTestDir2;
   private File qFileTestDir;
+  private File propertyDir;
 
   @Before
   public void setup() throws Exception {
@@ -56,6 +60,7 @@ public class TestTestParser {
         "test", "classes")));
     unitTestDir2 = Dirs.create(new File(baseDir, Joiner.on("/").join("source", "build", "2",
"units", "test", "classes")));
     qFileTestDir = Dirs.create(new File(baseDir, Joiner.on("/").join("source", "qfiles")));
+    propertyDir = Dirs.create(new File(baseDir, Joiner.on("/").join("source", "props")));
     Assert.assertTrue((new File(unitTestDir1, "TestA.class")).createNewFile());
     Assert.assertTrue((new File(unitTestDir2, "TestB.class")).createNewFile());
     Assert.assertTrue((new File(unitTestDir1, "TestC.class")).createNewFile());
@@ -64,10 +69,20 @@ public class TestTestParser {
     Assert.assertTrue((new File(qFileTestDir, ".svn")).mkdirs());
     Assert.assertTrue((new File(qFileTestDir, "dir.q")).mkdirs());
     Assert.assertTrue((new File(qFileTestDir, "normal.q")).createNewFile());
+    Assert.assertTrue((new File(qFileTestDir, "normal2.q")).createNewFile());
+    Assert.assertTrue((new File(qFileTestDir, "normal3.q")).createNewFile());
+    Assert.assertTrue((new File(qFileTestDir, "normal4.q")).createNewFile());
     Assert.assertTrue((new File(qFileTestDir, "excluded.q")).createNewFile());
     Assert.assertTrue((new File(qFileTestDir, "isolated.q")).createNewFile());
     Assert.assertTrue((new File(qFileTestDir, "included.q")).createNewFile());
 
+    Properties normalProp = new Properties();
+    normalProp.setProperty("normal.one.group", "normal.q,normal2.q");
+    normalProp.setProperty("normal.two.group", "normal3.q,normal4.q");
+    normalProp.setProperty("excluded.group", "excluded.q");
+    normalProp.setProperty("isolated.group", "isolated.q");
+    normalProp.setProperty("included.group", "included.q");
+    serialize("normal.properties", normalProp);
   }
   @After
   public void teardown() {
@@ -107,4 +122,35 @@ public class TestTestParser {
     List<TestBatch> testBatches = testParser.parse().get();
     Assert.assertEquals(4, testBatches.size());
   }
+  @Test
+  public void testParsePropertyFile() throws Exception {
+    context.put("unitTests.directories", "build/1 build/2");
+    context.put("unitTests.include", "TestA TestB");
+    context.put("unitTests.isolate", "TestB");
+    context.put("qFileTests", "f");
+    context.put("qFileTests.propertyFiles.prop",
+      "props" + File.separator + "normal.properties");
+    context.put("qFileTest.f.driver", DRIVER);
+    context.put("qFileTest.f.directory", "qfiles");
+    context.put("qFileTest.f.include", "included");
+    context.put("qFileTest.f.isolate", "isolated");
+    context.put("qFileTest.f.exclude", "excluded");
+    context.put("qFileTest.f.queryFilesProperty", "qfile");
+    context.put("qFileTest.f.groups.included", "prop.${normal.one.group} prop.${normal.two.group}
prop.${isolated.group}");
+    context.put("qFileTest.f.groups.isolated", "prop.${isolated.group}");
+    context.put("qFileTest.f.groups.excluded", "prop.${excluded.group}");
+    testParser = new TestParser(context, "testcase", workingDirectory, LOG);
+    List<TestBatch> testBatches = testParser.parse().get();
+    Assert.assertEquals(4, testBatches.size());
+  }
+
+  private void serialize(String propFileName, Properties props) throws Exception {
+    File f = new File(propertyDir, propFileName);
+    OutputStream out = new FileOutputStream(f);
+    try {
+      props.store(out, null);
+    } finally {
+      out.close();
+    }
+  }
 }



Mime
View raw message