Return-Path: X-Original-To: apmail-phoenix-commits-archive@minotaur.apache.org Delivered-To: apmail-phoenix-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C1FBD182D7 for ; Sat, 4 Jul 2015 15:22:57 +0000 (UTC) Received: (qmail 11929 invoked by uid 500); 4 Jul 2015 15:22:57 -0000 Delivered-To: apmail-phoenix-commits-archive@phoenix.apache.org Received: (qmail 11876 invoked by uid 500); 4 Jul 2015 15:22:57 -0000 Mailing-List: contact commits-help@phoenix.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@phoenix.apache.org Delivered-To: mailing list commits@phoenix.apache.org Received: (qmail 11702 invoked by uid 99); 4 Jul 2015 15:22:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Jul 2015 15:22:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 23E93E1806; Sat, 4 Jul 2015 15:22:56 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ravimagham@apache.org To: commits@phoenix.apache.org Date: Sat, 04 Jul 2015 15:23:00 -0000 Message-Id: <298a3b66b5144ad092d2c01ef9fe63ec@git.apache.org> In-Reply-To: <74d2bb760cb2405a86e4774048c169c1@git.apache.org> References: <74d2bb760cb2405a86e4774048c169c1@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [5/6] phoenix git commit: PHOENIX - 2036 PhoenixConfigurationUtil should provide a pre-normalize table name to PhoenixRuntime PHOENIX - 2036 PhoenixConfigurationUtil should provide a pre-normalize table name to PhoenixRuntime Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/2d1d9fcb Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/2d1d9fcb Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/2d1d9fcb Branch: refs/heads/4.x-HBase-0.98 Commit: 2d1d9fcbb4ce452ba08f0c3f8cf3343d4c6ded7f Parents: 86be0ba Author: ravimagham Authored: Sat Jul 4 08:21:13 2015 -0700 Committer: ravimagham Committed: Sat Jul 4 08:21:13 2015 -0700 ---------------------------------------------------------------------- .../org/apache/phoenix/util/PhoenixRuntime.java | 3 +- .../java/org/apache/phoenix/util/QueryUtil.java | 4 +- .../org/apache/phoenix/util/SchemaUtil.java | 18 ++++++- .../phoenix/pig/PhoenixHBaseLoaderIT.java | 9 ++-- .../phoenix/pig/PhoenixHBaseStorerIT.java | 4 +- .../apache/phoenix/pig/PhoenixHBaseStorage.java | 51 ++++++++------------ 6 files changed, 47 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/2d1d9fcb/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java index 18f914e..b7e1d85 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java @@ -349,8 +349,7 @@ public class PhoenixRuntime { public static List generateColumnInfo(Connection conn, String tableName, List columns) throws SQLException { - - PTable table = PhoenixRuntime.getTable(conn, tableName); + PTable table = PhoenixRuntime.getTable(conn, SchemaUtil.normalizeFullTableName(tableName)); List columnInfoList = Lists.newArrayList(); Set unresolvedColumnNames = new TreeSet(); if (columns == null) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/2d1d9fcb/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java index 2ef1c5f..9ec0efc 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java @@ -202,8 +202,6 @@ public final class QueryUtil { if(columnInfos == null || columnInfos.isEmpty()) { throw new IllegalArgumentException("At least one column must be provided"); } - // escape the table name to ensure it is case sensitive. - final String escapedFullTableName = SchemaUtil.getEscapedFullTableName(fullTableName); StringBuilder query = new StringBuilder(); query.append("SELECT "); for (ColumnInfo cinfo : columnInfos) { @@ -216,7 +214,7 @@ public final class QueryUtil { // Remove the trailing comma query.setLength(query.length() - 1); query.append(" FROM "); - query.append(escapedFullTableName); + query.append(fullTableName); if(conditions != null && conditions.length() > 0) { query.append(" WHERE (").append(conditions).append(")"); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/2d1d9fcb/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java index aff6b51..c674140 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java @@ -21,9 +21,8 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Strings.isNullOrEmpty; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME_BYTES; - +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES; import java.sql.SQLException; import java.sql.Statement; @@ -174,6 +173,21 @@ public class SchemaUtil { } return name.toUpperCase(); } + + /** + * Normalizes the fulltableName . Uses {@linkplain normalizeIdentifier} + * @param fullTableName + * @return + */ + public static String normalizeFullTableName(String fullTableName) { + String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName); + String tableName = SchemaUtil.getTableNameFromFullName(fullTableName); + String normalizedTableName = StringUtil.EMPTY_STRING; + if(!schemaName.isEmpty()) { + normalizedTableName = normalizeIdentifier(schemaName) + QueryConstants.NAME_SEPARATOR; + } + return normalizedTableName + normalizeIdentifier(tableName); + } public static boolean isCaseSensitive(String name) { return name!=null && name.length() > 0 && name.charAt(0)=='"'; http://git-wip-us.apache.org/repos/asf/phoenix/blob/2d1d9fcb/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java b/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java index 06c9bc1..3ac7843 100644 --- a/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java +++ b/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseLoaderIT.java @@ -74,6 +74,8 @@ public class PhoenixHBaseLoaderIT { private static final String TABLE_NAME = "A"; private static final String INDEX_NAME = "I"; private static final String TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME, TABLE_NAME); + private static final String CASE_SENSITIVE_TABLE_NAME = SchemaUtil.getEscapedArgument("a"); + private static final String CASE_SENSITIVE_TABLE_FULL_NAME = SchemaUtil.getTableName(SCHEMA_NAME,CASE_SENSITIVE_TABLE_NAME); private static HBaseTestingUtility hbaseTestUtil; private static String zkQuorum; private static Connection conn; @@ -234,13 +236,13 @@ public class PhoenixHBaseLoaderIT { public void testDataForTable() throws Exception { //create the table - String ddl = "CREATE TABLE " + TABLE_FULL_NAME + String ddl = "CREATE TABLE " + CASE_SENSITIVE_TABLE_FULL_NAME + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, AGE INTEGER) "; conn.createStatement().execute(ddl); //prepare data with 10 rows having age 25 and the other 30. - final String dml = "UPSERT INTO " + TABLE_FULL_NAME + " VALUES(?,?,?)"; + final String dml = "UPSERT INTO " + CASE_SENSITIVE_TABLE_FULL_NAME + " VALUES(?,?,?)"; PreparedStatement stmt = conn.prepareStatement(dml); int rows = 20; for(int i = 0 ; i < rows; i++) { @@ -253,7 +255,7 @@ public class PhoenixHBaseLoaderIT { //load data and filter rows whose age is > 25 pigServer.registerQuery(String.format( - "A = load 'hbase://table/%s' using " + PhoenixHBaseLoader.class.getName() + "('%s');", TABLE_FULL_NAME, + "A = load 'hbase://table/%s' using " + PhoenixHBaseLoader.class.getName() + "('%s');", CASE_SENSITIVE_TABLE_FULL_NAME, zkQuorum)); pigServer.registerQuery("B = FILTER A BY AGE > 25;"); @@ -605,6 +607,7 @@ public class PhoenixHBaseLoaderIT { @After public void tearDown() throws Exception { dropTable(TABLE_FULL_NAME); + dropTable(CASE_SENSITIVE_TABLE_FULL_NAME); pigServer.shutdown(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/2d1d9fcb/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseStorerIT.java ---------------------------------------------------------------------- diff --git a/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseStorerIT.java b/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseStorerIT.java index 67abf62..9106cdd 100644 --- a/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseStorerIT.java +++ b/phoenix-pig/src/it/java/org/apache/phoenix/pig/PhoenixHBaseStorerIT.java @@ -33,6 +33,7 @@ import java.util.Collection; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.util.SchemaUtil; import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS; @@ -140,12 +141,11 @@ public class PhoenixHBaseStorerIT extends BaseHBaseManagedTimeIT { */ @Test public void testStorerForSpecificColumns() throws Exception { - final String tableName = "TABLE2"; + final String tableName = SchemaUtil.getTableName("TABLE2", SchemaUtil.getEscapedArgument("zo2")); final Statement stmt = conn.createStatement(); stmt.execute("CREATE TABLE " + tableName + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, AGE INTEGER)"); - final Data data = Storage.resetData(pigServer); final Collection list = Lists.newArrayList(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/2d1d9fcb/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java ---------------------------------------------------------------------- diff --git a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java index 72d958b..4ada303 100644 --- a/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java +++ b/phoenix-pig/src/main/java/org/apache/phoenix/pig/PhoenixHBaseStorage.java @@ -18,8 +18,6 @@ package org.apache.phoenix.pig; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.sql.SQLException; import java.util.List; import java.util.Properties; @@ -33,12 +31,14 @@ import org.apache.commons.cli.ParseException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.OutputFormat; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.phoenix.mapreduce.PhoenixOutputFormat; import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; +import org.apache.phoenix.pig.util.TableSchemaParserFunction; import org.apache.phoenix.pig.writable.PhoenixPigDBWritable; import org.apache.phoenix.util.ColumnInfo; import org.apache.pig.ResourceSchema; @@ -88,12 +88,12 @@ public class PhoenixHBaseStorage implements StoreFuncInterface { private ResourceSchema schema; private long batchSize; private final PhoenixOutputFormat outputFormat = new PhoenixOutputFormat(); - // Set of options permitted private final static Options validOptions = new Options(); private final static CommandLineParser parser = new GnuParser(); private final static String SCHEMA = "_schema"; - + private final static String PHOENIX_TABLE_NAME_SCHEME = "hbase://"; + private final CommandLine configuredOptions; private final String server; @@ -134,33 +134,24 @@ public class PhoenixHBaseStorage implements StoreFuncInterface { */ @Override public void setStoreLocation(String location, Job job) throws IOException { - URI locationURI; - try { - locationURI = new URI(location); - if (!"hbase".equals(locationURI.getScheme())) { - throw new IOException(String.format("Location must use the hbase protocol, hbase://tableName[/columnList]. Supplied location=%s",location)); - } - - PhoenixConfigurationUtil.loadHBaseConfiguration(job); - config = job.getConfiguration(); - config.set(HConstants.ZOOKEEPER_QUORUM, server); - String tableName = locationURI.getAuthority(); - // strip off the leading path token '/' - String columns = null; - if(!locationURI.getPath().isEmpty()) { - columns = locationURI.getPath().substring(1); - PhoenixConfigurationUtil.setUpsertColumnNames(config, columns); - } - PhoenixConfigurationUtil.setOutputTableName(config,tableName); - PhoenixConfigurationUtil.setBatchSize(config,batchSize); - String serializedSchema = getUDFProperties().getProperty(contextSignature + SCHEMA); - if (serializedSchema != null) { - schema = (ResourceSchema) ObjectSerializer.deserialize(serializedSchema); - } - } catch (URISyntaxException e) { - throw new IOException(String.format("Location must use the hbase protocol, hbase://tableName[/columnList]. Supplied location=%s",location),e); + String tableSchema = location.substring(PHOENIX_TABLE_NAME_SCHEME.length()); + final TableSchemaParserFunction parseFunction = new TableSchemaParserFunction(); + Pair pair = parseFunction.apply(tableSchema); + PhoenixConfigurationUtil.loadHBaseConfiguration(job); + config = job.getConfiguration(); + config.set(HConstants.ZOOKEEPER_QUORUM, server); + String tableName = pair.getFirst(); + String columns = pair.getSecond(); + if(columns != null && columns.length() > 0) { + PhoenixConfigurationUtil.setUpsertColumnNames(config, columns); } - } + PhoenixConfigurationUtil.setOutputTableName(config,tableName); + PhoenixConfigurationUtil.setBatchSize(config,batchSize); + String serializedSchema = getUDFProperties().getProperty(contextSignature + SCHEMA); + if (serializedSchema != null) { + schema = (ResourceSchema) ObjectSerializer.deserialize(serializedSchema); + } + } @SuppressWarnings("unchecked") @Override