ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From niti...@apache.org
Subject ambari git commit: AMBARI-20113 : hive20 view : fixed : Not able to compute the table statistics for partitioned table (nitirajrathore)
Date Wed, 01 Mar 2017 18:06:51 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 8bcc953df -> a749f97ab


AMBARI-20113 : hive20 view : fixed : Not able to compute the table statistics for partitioned
table (nitirajrathore)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a749f97a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a749f97a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a749f97a

Branch: refs/heads/trunk
Commit: a749f97abbc44217fbb6f8e4868deaf6c4811fab
Parents: 8bcc953
Author: Nitiraj Singh Rathore <nitiraj.rathore@gmail.com>
Authored: Wed Mar 1 23:30:35 2017 +0530
Committer: Nitiraj Singh Rathore <nitiraj.rathore@gmail.com>
Committed: Wed Mar 1 23:30:35 2017 +0530

----------------------------------------------------------------------
 .../generators/AnalyzeTableQueryGenerator.java  |  49 ++++-
 .../view/hive20/resources/browser/DDLProxy.java |   6 +-
 .../hive20/resources/browser/DDLService.java    |   4 +-
 .../AnalyzeTableQueryGeneratorSpecTest.groovy   | 194 +++++++++++++++++++
 4 files changed, 242 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java
b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java
index 1d68407..dbe9d43 100644
--- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java
+++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGenerator.java
@@ -18,23 +18,58 @@
 
 package org.apache.ambari.view.hive20.internal.query.generators;
 
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
 import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
 import org.apache.ambari.view.hive20.exceptions.ServiceException;
+import org.apache.ambari.view.hive20.internal.dto.ColumnInfo;
+import org.apache.ambari.view.hive20.internal.dto.TableMeta;
+
+import javax.annotation.Nullable;
+
+import static org.apache.ambari.view.hive20.internal.query.generators.QueryGenerationUtils.isNullOrEmpty;
 
 public class AnalyzeTableQueryGenerator implements QueryGenerator {
-  private final String databaseName;
-  private final String tableName;
+  private TableMeta tableMeta;
   private final Boolean shouldAnalyzeColumns;
 
-  public AnalyzeTableQueryGenerator(String databaseName, String tableName, Boolean shouldAnalyzeColumns)
{
-    this.databaseName = databaseName;
-    this.tableName = tableName;
+  public AnalyzeTableQueryGenerator(TableMeta tableMeta, Boolean shouldAnalyzeColumns) {
+    this.tableMeta = tableMeta;
     this.shouldAnalyzeColumns = shouldAnalyzeColumns;
   }
 
   @Override
   public Optional<String> getQuery() throws ServiceException {
-    return Optional.of("ANALYZE TABLE " + "`" + databaseName + "`.`" + tableName + "`" +
" COMPUTE STATISTICS " +
-      (shouldAnalyzeColumns? " FOR COLUMNS ": "") + ";");
+    StringBuilder query = new StringBuilder("ANALYZE TABLE " );
+    query.append("`").append(tableMeta.getDatabase()).append("`").append(".").append("`").append(tableMeta.getTable()).append("`");
+
+    if( null != tableMeta.getPartitionInfo() && !isNullOrEmpty(tableMeta.getPartitionInfo().getColumns())){
+      query.append(" PARTITION (")
+           .append(Joiner.on(",")
+              .join(FluentIterable.from(tableMeta.getPartitionInfo().getColumns())
+                  .transform(
+                  new Function<ColumnInfo, Object>() {
+                    @Nullable
+                    @Override
+                    public Object apply(@Nullable ColumnInfo columnInfo) {
+                      return columnInfo.getName();
+                    }
+                  })
+              )
+           )
+           .append(")");
+    }
+
+
+    query.append(" COMPUTE STATISTICS ");
+
+    if(shouldAnalyzeColumns){
+      query.append(" FOR COLUMNS ");
+    }
+
+    query.append(";");
+
+    return Optional.of(query.toString());
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java
b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java
index e433dc4..0c93ba3 100644
--- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java
+++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLProxy.java
@@ -332,8 +332,10 @@ public class DDLProxy {
     }
   }
 
-  public Job analyzeTable(String databaseName, String tableName, Boolean shouldAnalyzeColumns,
JobResourceManager resourceManager) throws ServiceException {
-    AnalyzeTableQueryGenerator queryGenerator = new AnalyzeTableQueryGenerator(databaseName,
tableName, shouldAnalyzeColumns);
+  public Job analyzeTable(String databaseName, String tableName, Boolean shouldAnalyzeColumns,
JobResourceManager resourceManager, ConnectionConfig hiveConnectionConfig) throws ServiceException
{
+    TableMeta tableMeta = this.getTableProperties(context, hiveConnectionConfig, databaseName,
tableName);
+
+    AnalyzeTableQueryGenerator queryGenerator = new AnalyzeTableQueryGenerator(tableMeta,
shouldAnalyzeColumns);
     Optional<String> analyzeTable = queryGenerator.getQuery();
     String jobTitle = "Analyze table " + databaseName + "." + tableName;
     if(analyzeTable.isPresent()) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java
b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java
index 89b9d84..f5a4781 100644
--- a/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java
+++ b/contrib/views/hive20/src/main/java/org/apache/ambari/view/hive20/resources/browser/DDLService.java
@@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory;
 import javax.inject.Inject;
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
-import javax.ws.rs.FormParam;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
@@ -184,7 +183,8 @@ public class DDLService extends BaseService {
       shouldAnalyzeColumns = Boolean.valueOf(analyzeColumns.trim());
     }
     try {
-      Job job = proxy.analyzeTable(databaseName, tableName, shouldAnalyzeColumns, getResourceManager());
+      ConnectionConfig hiveConnectionConfig = getHiveConnectionConfig();
+      Job job = proxy.analyzeTable(databaseName, tableName, shouldAnalyzeColumns, getResourceManager(),
hiveConnectionConfig);
       JSONObject response = new JSONObject();
       response.put("job", job);
       return Response.status(Response.Status.ACCEPTED).entity(response).build();

http://git-wip-us.apache.org/repos/asf/ambari/blob/a749f97a/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy
b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy
new file mode 100644
index 0000000..7bc5850
--- /dev/null
+++ b/contrib/views/hive20/src/test/java/org/apache/ambari/view/hive20/internal/query/generators/AnalyzeTableQueryGeneratorSpecTest.groovy
@@ -0,0 +1,194 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ambari.view.hive20.internal.query.generators;
+
+import com.google.gson.Gson;
+import org.apache.ambari.view.hive20.internal.dto.TableMeta;
+import spock.lang.Specification;
+import com.google.common.base.Optional;
+
+class AnalyzeTableQueryGeneratorSpecTest extends Specification {
+  def "analyze with partition and for columns"() {
+    setup:
+    String tableMetaJson = "{" +
+            "\"database\": \"d1\"," +
+            "\"table\": \"t2\"," +
+            "\"columns\": [{" +
+            "\"name\": \"col_name1\"," +
+            "\"type\": \"string\"," +
+            "\"comment\": \"col_name1 comment\"" +
+            "}, {" +
+            "\"name\": \"col_name2\"," +
+            "\"type\": \"decimal(10,2)\"," +
+            "\"comment\": \"col_name2 comment\"" +
+            "}]," +
+            "\"partitionInfo\": {" +
+            "\"columns\": [{" +
+            "\"name\": \"col_name4\"," +
+            "\"type\": \"char(1)\"," +
+            "\"comment\": \"col_name4 comment\"" +
+            "}, {" +
+            "\"name\": \"col_name3\"," +
+            "\"type\": \"string\"," +
+            "\"comment\": \"col_name3 comment\"" +
+            "}]" +
+            "}" +
+            "}";
+
+    TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class);
+    AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, true);
+
+    when:
+    Optional<String> databaseDeleteQuery = generator.getQuery()
+
+    then:
+    databaseDeleteQuery.isPresent()
+
+    when:
+    String query = databaseDeleteQuery.get();
+
+    then:
+    query == "ANALYZE TABLE `d1`.`t2` PARTITION (col_name4,col_name3) COMPUTE STATISTICS
 FOR COLUMNS ;"
+  }
+  def "analyze with partition"() {
+    setup:
+    String tableMetaJson = "{" +
+            "\"database\": \"d1\"," +
+            "\"table\": \"t2\"," +
+            "\"columns\": [{" +
+            "\"name\": \"col_name1\"," +
+            "\"type\": \"string\"," +
+            "\"comment\": \"col_name1 comment\"" +
+            "}, {" +
+            "\"name\": \"col_name2\"," +
+            "\"type\": \"decimal(10,2)\"," +
+            "\"comment\": \"col_name2 comment\"" +
+            "}]," +
+            "\"partitionInfo\": {" +
+            "\"columns\": [{" +
+            "\"name\": \"col_name4\"," +
+            "\"type\": \"char(1)\"," +
+            "\"comment\": \"col_name4 comment\"" +
+            "}, {" +
+            "\"name\": \"col_name3\"," +
+            "\"type\": \"string\"," +
+            "\"comment\": \"col_name3 comment\"" +
+            "}]" +
+            "}" +
+            "}";
+
+    TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class);
+    AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, false);
+
+    when:
+    Optional<String> databaseDeleteQuery = generator.getQuery()
+
+    then:
+    databaseDeleteQuery.isPresent()
+
+    when:
+    String query = databaseDeleteQuery.get();
+
+    then:
+    query == "ANALYZE TABLE `d1`.`t2` PARTITION (col_name4,col_name3) COMPUTE STATISTICS
;"
+  }
+
+  def "analyze without partition"() {
+    setup:
+    String tableMetaJson = "{" +
+            "\"database\": \"d1\"," +
+            "\"table\": \"t2\"," +
+            "\"columns\": [{" +
+            "\"name\": \"col_name1\"," +
+            "\"type\": \"string\"," +
+            "\"comment\": \"col_name1 comment\"" +
+            "}, {" +
+            "\"name\": \"col_name2\"," +
+            "\"type\": \"decimal(10,2)\"," +
+            "\"comment\": \"col_name2 comment\"" +
+            "}," +
+            "{" +
+            "\"name\": \"col_name4\"," +
+            "\"type\": \"char(1)\"," +
+            "\"comment\": \"col_name4 comment\"" +
+            "}, {" +
+            "\"name\": \"col_name3\"," +
+            "\"type\": \"string\"," +
+            "\"comment\": \"col_name3 comment\"" +
+            "}" +
+            "]" +
+            "}";
+
+    TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class);
+    AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, true);
+
+    when:
+    Optional<String> databaseDeleteQuery = generator.getQuery()
+
+    then:
+    databaseDeleteQuery.isPresent()
+
+    when:
+    String query = databaseDeleteQuery.get();
+
+    then:
+    query == "ANALYZE TABLE `d1`.`t2` COMPUTE STATISTICS  FOR COLUMNS ;"
+  }
+
+  def "analyze for table only"() {
+    setup:
+    String tableMetaJson = "{" +
+            "\"database\": \"d1\"," +
+            "\"table\": \"t2\"," +
+            "\"columns\": [{" +
+            "\"name\": \"col_name1\"," +
+            "\"type\": \"string\"," +
+            "\"comment\": \"col_name1 comment\"" +
+            "}, {" +
+            "\"name\": \"col_name2\"," +
+            "\"type\": \"decimal(10,2)\"," +
+            "\"comment\": \"col_name2 comment\"" +
+            "}," +
+            "{" +
+            "\"name\": \"col_name4\"," +
+            "\"type\": \"char(1)\"," +
+            "\"comment\": \"col_name4 comment\"" +
+            "}, {" +
+            "\"name\": \"col_name3\"," +
+            "\"type\": \"string\"," +
+            "\"comment\": \"col_name3 comment\"" +
+            "}" +
+            "]" +
+            "}";
+
+    TableMeta tableMeta = new Gson().fromJson(tableMetaJson, TableMeta.class);
+    AnalyzeTableQueryGenerator generator = new AnalyzeTableQueryGenerator(tableMeta, false);
+
+    when:
+    Optional<String> databaseDeleteQuery = generator.getQuery()
+
+    then:
+    databaseDeleteQuery.isPresent()
+
+    when:
+    String query = databaseDeleteQuery.get();
+
+    then:
+    query == "ANALYZE TABLE `d1`.`t2` COMPUTE STATISTICS ;"
+  }
+}


Mime
View raw message