phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tdsi...@apache.org
Subject [26/50] [abbrv] phoenix git commit: PHOENIX-2337 - Pherf - Prefix values override rule values for writes
Date Tue, 10 Nov 2015 01:18:41 GMT
PHOENIX-2337 - Pherf - Prefix values override rule values for writes


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/caa0dbb4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/caa0dbb4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/caa0dbb4

Branch: refs/heads/txn
Commit: caa0dbb46a47945eaa384b31f7a2ba8e9c7a6c08
Parents: 28007f8
Author: Cody Marcel <cody.marcel@gmail.com>
Authored: Mon Oct 19 13:25:15 2015 -0700
Committer: Cody Marcel <cmarcel@cmarcel-wsl1.internal.salesforce.com>
Committed: Tue Oct 20 13:55:12 2015 -0700

----------------------------------------------------------------------
 .../phoenix/pherf/rules/RulesApplier.java       | 70 ++++++++++++++------
 .../apache/phoenix/pherf/util/PhoenixUtil.java  | 23 +++----
 .../apache/phoenix/pherf/RuleGeneratorTest.java | 51 ++++++++++++--
 .../test/resources/datamodel/test_schema.sql    |  1 +
 .../test/resources/scenario/test_scenario.xml   | 14 +++-
 5 files changed, 116 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
index 4801081..06a046d 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java
@@ -19,6 +19,7 @@
 package org.apache.phoenix.pherf.rules;
 
 import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.math3.random.RandomDataGenerator;
 import org.apache.phoenix.pherf.PherfConstants;
 import org.apache.commons.lang.RandomStringUtils;
@@ -97,13 +98,6 @@ public class RulesApplier {
                 Column columnRule = getColumnForRule(ruleList, phxMetaColumn);
 
                 value = getDataValue(columnRule);
-                synchronized (value) {
-                    // Add the prefix to the value if it exists.
-                    if (columnRule.getPrefix() != null) {
-                        value.setValue(columnRule.getPrefix() + value.getValue());
-                    }
-                }
-
             } else {
                 logger.warn("Attempted to apply rule to data, but could not find a rule to
match type:"
                                 + phxMetaColumn.getType()
@@ -122,46 +116,55 @@ public class RulesApplier {
      */
     public DataValue getDataValue(Column column) throws Exception{
         DataValue data = null;
+        String prefix = "";
         int length = column.getLength();
         int nullChance = column.getNullChance();
         List<DataValue> dataValues = column.getDataValues();
 
-        // Return an empty value if we we fall within the configured probability
+        // Return an empty value if we fall within the configured probability of null
         if ((nullChance != Integer.MIN_VALUE) && (isValueNull(nullChance))) {
             return new DataValue(column.getType(), "");
         }
 
+        if (column.getPrefix() != null) {
+            prefix = column.getPrefix();
+        }
+
+        if (prefix.length() >= length) {
+            logger.warn("You are attempting to generate data with a prefix (" + prefix +
") "
+                    + "That is longer than expected overall field length (" + length + ").
"
+                    + "This will certainly lead to unexpected data values.");
+        }
+
         switch (column.getType()) {
             case VARCHAR:
                 // Use the specified data values from configs if they exist
                 if ((column.getDataValues() != null) && (column.getDataValues().size()
> 0)) {
-                    data = generateDataValue(dataValues);
+                    data = pickDataValueFromList(dataValues);
                 } else {
                     Preconditions.checkArgument(length > 0, "length needs to be > 0");
                     if (column.getDataSequence() == DataSequence.SEQUENTIAL) {
                         data = getSequentialDataValue(column);
                     } else {
-                        String varchar = RandomStringUtils.randomAlphanumeric(length);
-                        data = new DataValue(column.getType(), varchar);
+                        data = getRandomDataValue(column);
                     }
                 }
                 break;
             case CHAR:
                 if ((column.getDataValues() != null) && (column.getDataValues().size()
> 0)) {
-                    data = generateDataValue(dataValues);
+                    data = pickDataValueFromList(dataValues);
                 } else {
                     Preconditions.checkArgument(length > 0, "length needs to be > 0");
                     if (column.getDataSequence() == DataSequence.SEQUENTIAL) {
                         data = getSequentialDataValue(column);
                     } else {
-                        String varchar = RandomStringUtils.randomAlphanumeric(length);
-                        data = new DataValue(column.getType(), varchar);
+                        data = getRandomDataValue(column);
                     }
                 }
                 break;
             case DECIMAL:
                 if ((column.getDataValues() != null) && (column.getDataValues().size()
> 0)) {
-                    data = generateDataValue(dataValues);
+                    data = pickDataValueFromList(dataValues);
                 } else {
                     int precision = column.getPrecision();
                     double minDbl = column.getMinValue();
@@ -181,7 +184,7 @@ public class RulesApplier {
                 break;
             case INTEGER:
                 if ((column.getDataValues() != null) && (column.getDataValues().size()
> 0)) {
-                    data = generateDataValue(dataValues);
+                    data = pickDataValueFromList(dataValues);
                 } else {
                     int minInt = column.getMinValue();
                     int maxInt = column.getMaxValue();
@@ -192,7 +195,7 @@ public class RulesApplier {
                 break;
             case DATE:
                 if ((column.getDataValues() != null) && (column.getDataValues().size()
> 0)) {
-                    data = generateDataValue(dataValues);
+                    data = pickDataValueFromList(dataValues);
                 } else {
                     int minYear = column.getMinValue();
                     int maxYear = column.getMaxValue();
@@ -205,7 +208,8 @@ public class RulesApplier {
             default:
                 break;
         }
-        Preconditions.checkArgument(data != null, "Data value could not be generated for
some reason. Please check configs");
+        Preconditions.checkArgument(data != null,
+                "Data value could not be generated for some reason. Please check configs");
         return data;
     }
 
@@ -246,7 +250,7 @@ public class RulesApplier {
         return (rndNull.nextInt(100) < chance);
     }
 
-    private DataValue generateDataValue(List<DataValue> values) throws Exception{
+    private DataValue pickDataValueFromList(List<DataValue> values) throws Exception{
         DataValue generatedDataValue = null;
         int sum = 0, count = 0;
 
@@ -265,7 +269,7 @@ public class RulesApplier {
             int rndIndex = rndVal.nextInt(values.size());
             DataValue valueRule = values.get(rndIndex);
 
-            generatedDataValue = generateDataValue(valueRule);
+            generatedDataValue = pickDataValueFromList(valueRule);
 
             // While it's possible to get here if you have a bunch of really small distributions,
             // It's just really unlikely. This is just a safety just so we actually pick
a value.
@@ -278,7 +282,7 @@ public class RulesApplier {
         return generatedDataValue;
     }
 
-    private DataValue generateDataValue(final DataValue valueRule) throws Exception{
+    private DataValue pickDataValueFromList(final DataValue valueRule) throws Exception{
         DataValue retValue = new DataValue(valueRule);
 
         // Path taken when configuration specifies a specific value to be taken with the
<value> tag
@@ -339,6 +343,14 @@ public class RulesApplier {
         }
     }
 
+    public Column getRule(Column phxMetaColumn) {
+        // Assume the first rule map
+        Map<DataTypeMapping, List> ruleMap = modelList.get(0);
+
+        List<Column> ruleList = ruleMap.get(phxMetaColumn.getType());
+        return getColumnForRule(ruleList, phxMetaColumn);
+    }
+
     private Column getColumnForRule(List<Column> ruleList, Column phxMetaColumn) {
 
         // Column pointer to head of list
@@ -371,7 +383,21 @@ public class RulesApplier {
         long inc = COUNTER.getAndIncrement();
         String strInc = String.valueOf(inc);
         String varchar = RandomStringUtils.randomAlphanumeric(column.getLength() - strInc.length());
-        data = new DataValue(column.getType(), strInc + varchar);
+        varchar = (column.getPrefix() != null) ? column.getPrefix() + strInc + varchar :
+                strInc + varchar;
+
+        // Truncate string back down if it exceeds length
+        varchar = StringUtils.left(varchar,column.getLength());
+        data = new DataValue(column.getType(), varchar);
         return data;
     }
+
+    private DataValue getRandomDataValue(Column column) {
+        String varchar = RandomStringUtils.randomAlphanumeric(column.getLength());
+        varchar = (column.getPrefix() != null) ? column.getPrefix() + varchar : varchar;
+
+        // Truncate string back down if it exceeds length
+        varchar = StringUtils.left(varchar, column.getLength());
+        return new DataValue(column.getType(), varchar);
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
index 1f610f6..064c604 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
@@ -33,7 +33,6 @@ import java.util.Properties;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
 
-// TODO This class needs to be cleanup up a bit. I just wanted to get an initial placeholder
in.
 public class PhoenixUtil {
     private static final Logger logger = LoggerFactory.getLogger(PhoenixUtil.class);
     private static String zookeeper;
@@ -111,7 +110,9 @@ public class PhoenixUtil {
             result = preparedStatement.execute();
             connection.commit();
         } finally {
-            preparedStatement.close();
+            if(preparedStatement != null) {
+                preparedStatement.close();
+            }
         }
         return result;
     }
@@ -125,9 +126,15 @@ public class PhoenixUtil {
             connection.commit();
         } catch (SQLException e) {
             e.printStackTrace();
+            if(preparedStatement != null) {
+                logger.error("Failed to apply schema. Statement (" + preparedStatement.toString()
+ ")",
+                        e.getMessage());
+            }
         } finally {
             try {
-                preparedStatement.close();
+                if (preparedStatement != null) {
+                    preparedStatement.close();
+                }
             } catch (SQLException e) {
                 e.printStackTrace();
             }
@@ -199,7 +206,7 @@ public class PhoenixUtil {
 
     public synchronized List<Column> getColumnsFromPhoenix(String schemaName, String
tableName,
             Connection connection) throws SQLException {
-        List<Column> columnList = new ArrayList<Column>();
+        List<Column> columnList = new ArrayList<>();
         ResultSet resultSet = null;
         try {
             resultSet = getColumnsMetaData(schemaName, tableName, connection);
@@ -293,14 +300,6 @@ public class PhoenixUtil {
         executeStatement("UPDATE STATISTICS " + tableName, scenario);
     }
 
-    public MonitorManager loadCustomMonitors(MonitorManager manager) throws Exception {
-        Properties
-                properties =
-                PherfConstants.create().getProperties(PherfConstants.PHERF_PROPERTIES, false);
-
-        return manager;
-    }
-    
     /**
      * Get explain plan for a query
      *

http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
index 936eedb..37bfd47 100644
--- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
+++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java
@@ -18,17 +18,13 @@
 
 package org.apache.phoenix.pherf;
 
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.phoenix.pherf.configuration.Column;
 import org.apache.phoenix.pherf.configuration.DataModel;
 import org.apache.phoenix.pherf.configuration.DataSequence;
@@ -43,6 +39,8 @@ import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
 import org.junit.Test;
 
+import static org.junit.Assert.*;
+
 public class RuleGeneratorTest {
     private static final String matcherScenario = PherfConstants.SCENARIO_ROOT_PATTERN +
".xml";
 
@@ -179,7 +177,8 @@ public class RuleGeneratorTest {
             }
         }
 
-        assertTrue("Expected count in increments did not match expected", testSet.size()
== (threadCount * increments));
+        assertTrue("Expected count in increments did not match expected",
+                testSet.size() == (threadCount * increments));
     }
 
     @Test
@@ -205,6 +204,46 @@ public class RuleGeneratorTest {
         }
     }
 
+    @Test
+    public void testRuleOverrides() throws Exception {
+        XMLConfigParser parser = new XMLConfigParser(".*test_scenario.xml");
+        WriteWorkload loader = new WriteWorkload(parser);
+        RulesApplier rulesApplier = loader.getRulesApplier();
+        Scenario scenario = parser.getScenarios().get(0);
+
+        // We should be able to find the correct rule based only on Type and Name combination
+        // Test CHAR
+        Column simPhxCol = new Column();
+        simPhxCol.setName("OTHER_ID");
+        simPhxCol.setType(DataTypeMapping.CHAR);
+
+        // Get the rule we expect to match
+        Column rule = rulesApplier.getRule(simPhxCol);
+        assertEquals("Did not find the correct rule.", rule.getName(), simPhxCol.getName());
+        assertEquals("Did not find the matching rule type.", rule.getType(), simPhxCol.getType());
+        assertEquals("Rule contains incorrect length.", rule.getLength(), 8);
+        assertEquals("Rule contains incorrect prefix.", rule.getPrefix(), "z0Oxx00");
+
+        DataValue value = rulesApplier.getDataForRule(scenario, simPhxCol);
+        assertEquals("Value returned does not match rule.", value.getValue().length(), 8);
+
+        // Test VARCHAR with RANDOM and prefix
+        simPhxCol.setName("OLDVAL_STRING");
+        simPhxCol.setType(DataTypeMapping.VARCHAR);
+
+        // Get the rule we expect to match
+        rule = rulesApplier.getRule(simPhxCol);
+        assertEquals("Did not find the correct rule.", rule.getName(), simPhxCol.getName());
+        assertEquals("Did not find the matching rule type.", rule.getType(), simPhxCol.getType());
+        assertEquals("Rule contains incorrect length.", rule.getLength(), 10);
+        assertEquals("Rule contains incorrect prefix.", rule.getPrefix(), "MYPRFX");
+
+        value = rulesApplier.getDataForRule(scenario, simPhxCol);
+        assertEquals("Value returned does not match rule.", value.getValue().length(), 10);
+        assertTrue("Value returned start with prefix.",
+                StringUtils.startsWith(value.getValue(), rule.getPrefix()));
+    }
+
     /**
      * Asserts that the value field is between the min/max value fields
      *

http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
index 4e6b9d4..2cf8e9b 100644
--- a/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
+++ b/phoenix-pherf/src/test/resources/datamodel/test_schema.sql
@@ -19,6 +19,7 @@ CREATE TABLE IF NOT EXISTS PHERF.TEST_TABLE (
     TENANT_ID CHAR(15) NOT NULL,
     PARENT_ID CHAR(15) NOT NULL,
     CREATED_DATE DATE NOT NULL,
+    OTHER_ID CHAR(15),
     FIELD VARCHAR,
     OLDVAL_STRING VARCHAR,
     NEWVAL_STRING VARCHAR,

http://git-wip-us.apache.org/repos/asf/phoenix/blob/caa0dbb4/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
index f23e71a..def55ca 100644
--- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
+++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
@@ -112,7 +112,7 @@
         <column>
             <!-- This column type defines what will generally happen to VARCHAR fields
unless they are explicitly defined or overridden elsewhere -->
             <type>VARCHAR</type>
-            <length>15</length>
+            <length>10</length>
             <userDefined>true</userDefined>
             <dataSequence>RANDOM</dataSequence>
             <name>OLDVAL_STRING</name>
@@ -123,7 +123,7 @@
             <type>VARCHAR</type>
             <length>15</length>
             <userDefined>true</userDefined>
-            <dataSequence>SEQUENTIAL</dataSequence>
+            <dataSequence>RANDOM</dataSequence>
             <name>NEWVAL_STRING</name>
             <prefix>TSTPRFX</prefix>
         </column>
@@ -145,7 +145,15 @@
                     <value>LMN</value>
                 </datavalue>
             </valuelist>            
-        </column>        
+        </column>
+        <column>
+            <type>CHAR</type>
+            <userDefined>true</userDefined>
+            <dataSequence>SEQUENTIAL</dataSequence>
+            <length>8</length>
+            <name>OTHER_ID</name>
+            <prefix>z0Oxx00</prefix>
+        </column>
     </datamapping>
     <scenarios>
         <scenario tableName="PHERF.TEST_TABLE" rowCount="100" name="testScenarioRW">


Mime
View raw message