phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From codymar...@apache.org
Subject phoenix git commit: PHOENIX-1791. Adding ability for Pherf Write Workloads to write to a multi-tenant view.
Date Mon, 10 Aug 2015 18:23:25 GMT
Repository: phoenix
Updated Branches:
  refs/heads/4.5-HBase-1.1 b7cd7e505 -> 93bd2c33c


PHOENIX-1791. Adding ability for Pherf Write Workloads to write to a multi-tenant view.


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/93bd2c33
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/93bd2c33
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/93bd2c33

Branch: refs/heads/4.5-HBase-1.1
Commit: 93bd2c33ccfa9fca0d6f52a76d7e3c4d14637109
Parents: b7cd7e5
Author: Jan <jfernando@salesforce.com>
Authored: Thu Aug 6 11:12:52 2015 -0700
Committer: Cody Marcel <cody.marcel@gmail.com>
Committed: Mon Aug 10 11:22:46 2015 -0700

----------------------------------------------------------------------
 .../org/apache/phoenix/pherf/DataIngestIT.java  | 74 +++++++++++++++++---
 .../phoenix/pherf/configuration/Scenario.java   | 25 +++++--
 .../pherf/configuration/XMLConfigParser.java    | 23 +++---
 .../apache/phoenix/pherf/util/PhoenixUtil.java  | 28 +++++---
 .../phoenix/pherf/workload/WriteWorkload.java   | 36 +++++-----
 .../datamodel/test_schema_mt_table.sql          | 31 ++++++++
 .../resources/datamodel/test_schema_mt_view.sql |  1 +
 .../test/resources/scenario/test_scenario.xml   | 22 ++++++
 8 files changed, 187 insertions(+), 53 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/93bd2c33/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
index 8a340b3..2b9d309 100644
--- a/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
+++ b/phoenix-pherf/src/it/java/org/apache/phoenix/pherf/DataIngestIT.java
@@ -18,8 +18,19 @@
 
 package org.apache.phoenix.pherf;
 
-import com.jcabi.jdbc.JdbcSession;
-import com.jcabi.jdbc.Outcome;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.phoenix.pherf.configuration.Column;
 import org.apache.phoenix.pherf.configuration.DataModel;
 import org.apache.phoenix.pherf.configuration.DataTypeMapping;
@@ -33,15 +44,8 @@ import org.apache.phoenix.pherf.workload.WriteWorkload;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.*;
+import com.jcabi.jdbc.JdbcSession;
+import com.jcabi.jdbc.Outcome;
 
 public class DataIngestIT extends ResultBaseTestIT {
 
@@ -144,4 +148,52 @@ public class DataIngestIT extends ResultBaseTestIT {
             fail("Failed to load data. An exception was thrown: " + e.getMessage());
         }
     }
+
+
+    @Test
+    /**
+     * Validates that Pherf can write data to a Multi-Tenant View in addition to 
+     * standard Phoenix tables.
+     */
+    public void testMultiTenantViewWriteWorkload() throws Exception {
+        // Arrange
+        Scenario scenario = parser.getScenarioByName("testMTWriteScenario");
+        WorkloadExecutor executor = new WorkloadExecutor();
+        executor.add(new WriteWorkload(util, parser, scenario));
+        
+        // Act
+        try {
+            // Wait for data to load up.
+            executor.get();
+            executor.shutdown();
+        } catch (Exception e) {
+            fail("Failed to load data. An exception was thrown: " + e.getMessage());
+        }
+
+        // Assert
+        assertExpectedNumberOfRecordsWritten(scenario);
+
+    }
+
+    private void assertExpectedNumberOfRecordsWritten(Scenario scenario) throws Exception,
+            SQLException {
+        Connection connection = util.getConnection(scenario.getTenantId());
+        String sql = "select count(*) from " + scenario.getTableName();
+        Integer count = new JdbcSession(connection).sql(sql).select(new Outcome<Integer>()
{
+            @Override public Integer handle(ResultSet resultSet, Statement statement)
+                    throws SQLException {
+                while (resultSet.next()) {
+                    return resultSet.getInt(1);
+                }
+                return null;
+            }
+        });
+        assertNotNull("Could not retrieve count. " + count);
+        assertTrue("Could not query any rows for in " + scenario.getTableName(), count >
0);
+        assertEquals("Expected 100 rows to have been inserted", scenario.getRowCount(), count.intValue());
+    }
+    
+
+
+
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/93bd2c33/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
index 7de96cc..6c949d8 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/Scenario.java
@@ -18,15 +18,16 @@
 
 package org.apache.phoenix.pherf.configuration;
 
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.phoenix.pherf.util.PhoenixUtil;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.phoenix.pherf.util.PhoenixUtil;
 
 @XmlRootElement(namespace = "org.apache.phoenix.pherf.configuration.DataModel")
 public class Scenario {
@@ -37,6 +38,7 @@ public class Scenario {
     private List<QuerySet> querySet = new ArrayList<>();
     private WriteParams writeParams;
     private String name;
+    private String tenantId;
 
     public Scenario() {
         writeParams = new WriteParams();
@@ -162,6 +164,19 @@ public class Scenario {
     public void setName(String name) {
         this.name = name;
     }
+    
+    /**
+     * Tenant Id used by connection of this query
+     * @return
+     */
+    @XmlAttribute
+    public String getTenantId() {
+        return tenantId;
+    }
+
+    public void setTenantId(String tenantId) {
+        this.tenantId = tenantId;
+    }
 
     public WriteParams getWriteParams() {
         return writeParams;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/93bd2c33/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
index 393fa7e..93dc94c 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/configuration/XMLConfigParser.java
@@ -18,16 +18,6 @@
 
 package org.apache.phoenix.pherf.configuration;
 
-import org.apache.phoenix.pherf.PherfConstants;
-import org.apache.phoenix.pherf.exception.FileLoaderException;
-import org.apache.phoenix.pherf.util.ResourceList;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Marshaller;
-import javax.xml.bind.Unmarshaller;
 import java.io.OutputStream;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -35,6 +25,17 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.phoenix.pherf.PherfConstants;
+import org.apache.phoenix.pherf.exception.FileLoaderException;
+import org.apache.phoenix.pherf.util.ResourceList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class XMLConfigParser {
 
     private static final Logger logger = LoggerFactory.getLogger(XMLConfigParser.class);
@@ -134,6 +135,8 @@ public class XMLConfigParser {
         if (fullTableName.contains(".")) {
             ret = fullTableName.substring(fullTableName.indexOf(".") + 1, fullTableName.length());
         }
+        // Remove any quotes that may be needed for multi-tenant tables
+        ret = ret.replaceAll("\"", "");
         return ret;
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/93bd2c33/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
index 0156149..48d3888 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/util/PhoenixUtil.java
@@ -18,20 +18,26 @@
 
 package org.apache.phoenix.pherf.util;
 
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+
 import org.apache.phoenix.pherf.PherfConstants;
 import org.apache.phoenix.pherf.configuration.Column;
 import org.apache.phoenix.pherf.configuration.DataTypeMapping;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.sql.*;
-import java.util.*;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.phoenix.pherf.configuration.Query;
 import org.apache.phoenix.pherf.configuration.QuerySet;
+import org.apache.phoenix.pherf.configuration.Scenario;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -82,11 +88,11 @@ public class PhoenixUtil {
         return DriverManager.getConnection(url, props);
     }
 
-    public boolean executeStatement(String sql) throws Exception {
+    public boolean executeStatement(String sql, Scenario scenario) throws Exception {
         Connection connection = null;
         boolean result = false;
         try {
-            connection = getConnection();
+            connection = getConnection(scenario.getTenantId());
             result = executeStatement(sql, connection);
         } finally {
             if (connection != null) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/93bd2c33/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
index 305521b..4e7fbfc 100644
--- a/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
+++ b/phoenix-pherf/src/main/java/org/apache/phoenix/pherf/workload/WriteWorkload.java
@@ -18,6 +18,21 @@
 
 package org.apache.phoenix.pherf.workload;
 
+import java.math.BigDecimal;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
 import org.apache.phoenix.pherf.PherfConstants;
 import org.apache.phoenix.pherf.configuration.Column;
 import org.apache.phoenix.pherf.configuration.Scenario;
@@ -34,17 +49,6 @@ import org.apache.phoenix.pherf.util.RowCalculator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.math.BigDecimal;
-import java.sql.*;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
 public class WriteWorkload implements Workload {
     private static final Logger logger = LoggerFactory.getLogger(WriteWorkload.class);
     private final PhoenixUtil pUtil;
@@ -158,7 +162,7 @@ public class WriteWorkload implements Workload {
         waitForBatches(dataLoadTimeSummary, scenario, start, writeBatches);
 
         // always update stats for Phoenix base tables
-        updatePhoenixStats(scenario.getTableName());
+        updatePhoenixStats(scenario.getTableName(), scenario);
     }
 
     private List<Future> getBatches(DataLoadThreadTime dataLoadThreadTime, Scenario
scenario)
@@ -172,7 +176,7 @@ public class WriteWorkload implements Workload {
             List<Column>
                     phxMetaCols =
                     pUtil.getColumnsFromPhoenix(scenario.getSchemaName(),
-                            scenario.getTableNameWithoutSchemaName(), pUtil.getConnection());
+                            scenario.getTableNameWithoutSchemaName(), pUtil.getConnection(scenario.getTenantId()));
             int threadRowCount = rowCalculator.getNext();
             logger.info(
                     "Kick off thread (#" + i + ")for upsert with (" + threadRowCount + ")
rows.");
@@ -215,9 +219,9 @@ public class WriteWorkload implements Workload {
      * @param tableName
      * @throws Exception
      */
-    public void updatePhoenixStats(String tableName) throws Exception {
+    public void updatePhoenixStats(String tableName, Scenario scenario) throws Exception
{
         logger.info("Updating stats for " + tableName);
-        pUtil.executeStatement("UPDATE STATISTICS " + tableName);
+        pUtil.executeStatement("UPDATE STATISTICS " + tableName, scenario);
     }
 
     public Future<Info> upsertData(final Scenario scenario, final List<Column>
columns,
@@ -230,7 +234,7 @@ public class WriteWorkload implements Workload {
                 SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                 Connection connection = null;
                 try {
-                    connection = pUtil.getConnection();
+                    connection = pUtil.getConnection(scenario.getTenantId());
                     long logStartTime = System.currentTimeMillis();
                     long
                             maxDuration =

http://git-wip-us.apache.org/repos/asf/phoenix/blob/93bd2c33/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql
new file mode 100644
index 0000000..b6791bf
--- /dev/null
+++ b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_table.sql
@@ -0,0 +1,31 @@
+/*
+  -- Licensed to the Apache Software Foundation (ASF) under one
+  -- or more contributor license agreements.  See the NOTICE file
+  -- distributed with this work for additional information
+  -- regarding copyright ownership.  The ASF licenses this file
+  -- to you under the Apache License, Version 2.0 (the
+  -- "License"); you may not use this file except in compliance
+  -- with the License.  You may obtain a copy of the License at
+  --
+  -- http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+*/
+CREATE TABLE IF NOT EXISTS PHERF.TEST_MULTI_TENANT_TABLE (
+    TENANT_ID CHAR(15) NOT NULL,
+    IDENTIFIER CHAR(3) NOT NULL,
+    ID CHAR(15) NOT NULL,
+    CREATED_DATE DATE,
+    FIELD VARCHAR,
+    SOME_INT INTEGER
+    CONSTRAINT PK PRIMARY KEY
+    (
+        TENANT_ID,
+        IDENTIFIER,
+        ID
+    )
+) VERSIONS=1,MULTI_TENANT=true

http://git-wip-us.apache.org/repos/asf/phoenix/blob/93bd2c33/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql
new file mode 100644
index 0000000..5f5d7ec
--- /dev/null
+++ b/phoenix-pherf/src/test/resources/datamodel/test_schema_mt_view.sql
@@ -0,0 +1 @@
+CREATE VIEW IF NOT EXISTS PHERF.TEST_VIEW (field1 VARCHAR, field2 VARCHAR) AS SELECT * FROM
PHERF.TEST_MULTI_TENANT_TABLE
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/93bd2c33/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
----------------------------------------------------------------------
diff --git a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
index fddf022..b5fe564 100644
--- a/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
+++ b/phoenix-pherf/src/test/resources/scenario/test_scenario.xml
@@ -127,6 +127,25 @@
             <name>NEWVAL_STRING</name>
             <prefix>TSTPRFX</prefix>
         </column>
+        <column>
+            <type>CHAR</type>
+            <length>3</length>
+            <userDefined>true</userDefined>
+            <dataSequence>LIST</dataSequence>
+            <name>IDENTIFIER</name>
+            <valuelist>
+                <!-- Distributes according to specified values. These must total 100 -->
+                <datavalue distribution="60">
+                    <value>ABC</value>
+                </datavalue>
+                <datavalue distribution="20">
+                    <value>XYZ</value>
+                </datavalue>
+                <datavalue distribution="20">
+                    <value>LMN</value>
+                </datavalue>
+            </valuelist>            
+        </column>        
     </datamapping>
     <scenarios>
         <scenario tableName="PHERF.TEST_TABLE" rowCount="100" name="testScenarioRW">
@@ -201,5 +220,8 @@
                 <query id="q4" statement="select sum(SOME_INT) from PHERF.TEST_TABLE"/>
             </querySet>
         </scenario>
+        <!-- Test writing to a Multi-tenant View -->
+        <scenario tableName="PHERF.TEST_VIEW" tenantId="abcdefghijklmno" rowCount="100"
name="testMTWriteScenario">
+        </scenario>
     </scenarios>
 </datamodel>
\ No newline at end of file


Mime
View raw message