Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 5384F200C2A for ; Wed, 1 Mar 2017 18:39:48 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 5224D160B70; Wed, 1 Mar 2017 17:39:48 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 49BC8160B56 for ; Wed, 1 Mar 2017 18:39:47 +0100 (CET) Received: (qmail 86884 invoked by uid 500); 1 Mar 2017 17:39:46 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 86875 invoked by uid 99); 1 Mar 2017 17:39:46 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 01 Mar 2017 17:39:46 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5FC93DFDAC; Wed, 1 Mar 2017 17:39:46 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: nitiraj@apache.org To: commits@ambari.apache.org Message-Id: <9a5bf80c72ba4a4b83f28cc82f44338a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-20113 : hive20 view : fixed : Not able to compute the table statistics for partitioned table (nitirajrathore) Date: Wed, 1 Mar 2017 17:39:46 +0000 (UTC) archived-at: Wed, 01 Mar 2017 17:39:48 -0000 Repository: ambari Updated Branches: refs/heads/trunk 340d36706 -> a72f3dda7 AMBARI-20113 : hive20 view : fixed : Not able to compute the table statistics for partitioned table (nitirajrathore) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a72f3dda Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a72f3dda Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a72f3dda Branch: refs/heads/trunk Commit: a72f3dda7992aeb717fbf71401b6f60253054c41 Parents: 340d367 Author: Nitiraj Singh Rathore Authored: Wed Mar 1 23:02:48 2017 +0530 Committer: Nitiraj Singh Rathore Committed: Wed Mar 1 23:05:22 2017 +0530 ---------------------------------------------------------------------- .../generators/AnalyzeTableQueryGenerator.java | 49 ++++- .../view/hive20/resources/browser/DDLProxy.java | 6 +- .../hive20/resources/browser/DDLService.java | 4 +- .../AnalyzeTableQueryGeneratorSpecTest.groovy | 193 +++++++++++++++++++ 4 files changed, 241 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a72f3dda/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java index 1d68407..dbe9d43 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java @@ -18,23 +18,58 @@ package org.apache.ambari.view.hive20.internal.query.generators; +import com.google.common.base.Function; +import com.google.common.base.Joiner; import com.google.common.base.Optional; +import com.google.common.collect.FluentIterable; import org.apache.ambari.view.hive20.exceptions.ServiceException; +import org.apache.ambari.view.hive20.internal.dto.ColumnInfo; +import org.apache.ambari.view.hive20.internal.dto.TableMeta; + +import javax.annotation.Nullable; + +import static org.apache.ambari.view.hive20.internal.query.generators.QueryGenerationUtils.isNullOrEmpty; public class AnalyzeTableQueryGenerator implements QueryGenerator { - private final String databaseName; - private final String tableName; + private TableMeta tableMeta; private final Boolean shouldAnalyzeColumns; - public AnalyzeTableQueryGenerator(String databaseName, String tableName, Boolean shouldAnalyzeColumns) { - this.databaseName = databaseName; - this.tableName = tableName; + public AnalyzeTableQueryGenerator(TableMeta tableMeta, Boolean shouldAnalyzeColumns) { + this.tableMeta = tableMeta; this.shouldAnalyzeColumns = shouldAnalyzeColumns; } @Override public Optional getQuery() throws ServiceException { - return Optional.of("ANALYZE TABLE " + "`" + databaseName + "`.`" + tableName + "`" + " COMPUTE STATISTICS " + - (shouldAnalyzeColumns? " FOR COLUMNS ": "") + ";"); + StringBuilder query = new StringBuilder("ANALYZE TABLE " ); + query.append("`").append(tableMeta.getDatabase()).append("`").append(".").append("`").append(tableMeta.getTable()).append("`"); + + if( null != tableMeta.getPartitionInfo() && !isNullOrEmpty(tableMeta.getPartitionInfo().getColumns())){ + query.append(" PARTITION (") + .append(Joiner.on(",") + .join(FluentIterable.from(tableMeta.getPartitionInfo().getColumns()) + .transform( + new Function() { + @Nullable + @Override + public Object apply(@Nullable ColumnInfo columnInfo) { + return columnInfo.getName(); + } + }) + ) + ) + .append(")"); + } + + + query.append(" COMPUTE STATISTICS "); + + if(shouldAnalyzeColumns){ + query.append(" FOR COLUMNS "); + } + + query.append(";"); + + return Optional.of(query.toString()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/a72f3dda/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java index e433dc4..0c93ba3 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java @@ -332,8 +332,10 @@ public class DDLProxy { } } - public Job analyzeTable(String databaseName, String tableName, Boolean shouldAnalyzeColumns, JobResourceManager resourceManager) throws ServiceException { - AnalyzeTableQueryGenerator queryGenerator = new AnalyzeTableQueryGenerator(databaseName, tableName, shouldAnalyzeColumns); + public Job analyzeTable(String databaseName, String tableName, Boolean shouldAnalyzeColumns, JobResourceManager resourceManager, ConnectionConfig hiveConnectionConfig) throws ServiceException { + TableMeta tableMeta = this.getTableProperties(context, hiveConnectionConfig, databaseName, tableName); + + AnalyzeTableQueryGenerator queryGenerator = new AnalyzeTableQueryGenerator(tableMeta, shouldAnalyzeColumns); Optional analyzeTable = queryGenerator.getQuery(); String jobTitle = "Analyze table " + databaseName + "." + tableName; if(analyzeTable.isPresent()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/a72f3dda/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java index 89b9d84..f5a4781 100644 --- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java +++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java @@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; -import javax.ws.rs.FormParam; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; @@ -184,7 +183,8 @@ public class DDLService extends BaseService { shouldAnalyzeColumns = Boolean.valueOf(analyzeColumns.trim()); } try { - Job job = proxy.analyzeTable(databaseName, tableName, shouldAnalyzeColumns, getResourceManager()); + ConnectionConfig hiveConnectionConfig = getHiveConnectionConfig(); + Job job = proxy.analyzeTable(databaseName, tableName, shouldAnalyzeColumns, getResourceManager(), hiveConnectionConfig); JSONObject response = new JSONObject(); response.put("job", job); return Response.status(Response.Status.ACCEPTED).entity(response).build(); http://git-wip-us.apache.org/repos/asf/ambari/blob/a72f3dda/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy ---------------------------------------------------------------------- diff --git a/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy new file mode 100644 index 0000000..91d1b44 --- /dev/null +++ b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy @@ -0,0 +1,193 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.ambari.view.hive20.internal.query.generators; + +import com.google.gson.Gson; +import org.apache.ambari.view.hive20.internal.dto.TableMeta; +import spock.lang.Specification; + +class AnalyzeTableQueryGeneratorSpecTest extends Specification { + def "analyze with partition and for columns"() { + setup: + String tableMetaJson = "{" + + "\"database\": \"d1\"," + + "\"table\": \"t2\"," + + "\"columns\": [{" + + "\"name\": \"col_name1\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name1 comment\"" + + "}, {" + + "\"name\": \"col_name2\"," + + "\"type\": \"decimal(10,2)\"," + + "\"comment\": \"col_name2 comment\"" + + "}]," + + "\"partitionInfo\": {" + + "\"columns\": [{" + + "\"name\": \"col_name4\"," + + "\"type\": \"char(1)\"," + + "\"comment\": \"col_name4 comment\"" + + "}, {" + + "\"name\": \"col_name3\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name3 comment\"" + + "}]" + + "}" + + "}"; + + TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class); + AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, true); + + when: + Optional databaseDeleteQuery = generator.getQuery() + + then: + databaseDeleteQuery.isPresent() + + when: + String query = databaseDeleteQuery.get(); + + then: + query == "ANALYZE TABLE `d1`.`t2` PARTITION (col_name4,col_name3) COMPUTE STATISTICS FOR COLUMNS ;" + } + def "analyze with partition"() { + setup: + String tableMetaJson = "{" + + "\"database\": \"d1\"," + + "\"table\": \"t2\"," + + "\"columns\": [{" + + "\"name\": \"col_name1\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name1 comment\"" + + "}, {" + + "\"name\": \"col_name2\"," + + "\"type\": \"decimal(10,2)\"," + + "\"comment\": \"col_name2 comment\"" + + "}]," + + "\"partitionInfo\": {" + + "\"columns\": [{" + + "\"name\": \"col_name4\"," + + "\"type\": \"char(1)\"," + + "\"comment\": \"col_name4 comment\"" + + "}, {" + + "\"name\": \"col_name3\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name3 comment\"" + + "}]" + + "}" + + "}"; + + TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class); + AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, false); + + when: + Optional databaseDeleteQuery = generator.getQuery() + + then: + databaseDeleteQuery.isPresent() + + when: + String query = databaseDeleteQuery.get(); + + then: + query == "ANALYZE TABLE `d1`.`t2` PARTITION (col_name4,col_name3) COMPUTE STATISTICS ;" + } + + def "analyze without partition"() { + setup: + String tableMetaJson = "{" + + "\"database\": \"d1\"," + + "\"table\": \"t2\"," + + "\"columns\": [{" + + "\"name\": \"col_name1\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name1 comment\"" + + "}, {" + + "\"name\": \"col_name2\"," + + "\"type\": \"decimal(10,2)\"," + + "\"comment\": \"col_name2 comment\"" + + "}," + + "{" + + "\"name\": \"col_name4\"," + + "\"type\": \"char(1)\"," + + "\"comment\": \"col_name4 comment\"" + + "}, {" + + "\"name\": \"col_name3\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name3 comment\"" + + "}" + + "]" + + "}"; + + TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class); + AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, true); + + when: + Optional databaseDeleteQuery = generator.getQuery() + + then: + databaseDeleteQuery.isPresent() + + when: + String query = databaseDeleteQuery.get(); + + then: + query == "ANALYZE TABLE `d1`.`t2` COMPUTE STATISTICS FOR COLUMNS ;" + } + + def "analyze for table only"() { + setup: + String tableMetaJson = "{" + + "\"database\": \"d1\"," + + "\"table\": \"t2\"," + + "\"columns\": [{" + + "\"name\": \"col_name1\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name1 comment\"" + + "}, {" + + "\"name\": \"col_name2\"," + + "\"type\": \"decimal(10,2)\"," + + "\"comment\": \"col_name2 comment\"" + + "}," + + "{" + + "\"name\": \"col_name4\"," + + "\"type\": \"char(1)\"," + + "\"comment\": \"col_name4 comment\"" + + "}, {" + + "\"name\": \"col_name3\"," + + "\"type\": \"string\"," + + "\"comment\": \"col_name3 comment\"" + + "}" + + "]" + + "}"; + + TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class); + AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, false); + + when: + Optional databaseDeleteQuery = generator.getQuery() + + then: + databaseDeleteQuery.isPresent() + + when: + String query = databaseDeleteQuery.get(); + + then: + query == "ANALYZE TABLE `d1`.`t2` COMPUTE STATISTICS ;" + } +}