Mailing-List: contact dev-help@drill.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@drill.apache.org
From: jinfengni <git@git.apache.org>
To: dev@drill.apache.org
Reply-To: dev@drill.apache.org
References: <git-pr-654-drill@git.apache.org>
In-Reply-To: <git-pr-654-drill@git.apache.org>
Subject: [GitHub] drill pull request #654: DRILL-5032: Drill query on hive parquet table faile...
Content-Type: text/plain
Message-Id: <20161215161754.52073EA922@git1-us-west.apache.org>
Date: Thu, 15 Dec 2016 16:17:54 +0000 (UTC)
archived-at: Thu, 15 Dec 2016 16:17:57 -0000

Github user jinfengni commented on a diff in the pull request:

    https://github.com/apache/drill/pull/654#discussion_r92307044
  
    --- Diff: contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveTableWrapper.java ---
    @@ -0,0 +1,483 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.drill.exec.store.hive;
    +
    +import java.util.ArrayList;
    +import java.util.HashMap;
    +import java.util.List;
    +import java.util.Map;
    +
    +import org.apache.hadoop.hive.metastore.api.FieldSchema;
    +import org.apache.hadoop.hive.metastore.api.Order;
    +import org.apache.hadoop.hive.metastore.api.Partition;
    +import org.apache.hadoop.hive.metastore.api.SerDeInfo;
    +import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
    +
    +import com.fasterxml.jackson.annotation.JsonCreator;
    +import com.fasterxml.jackson.annotation.JsonIgnore;
    +import com.fasterxml.jackson.annotation.JsonProperty;
    +import com.fasterxml.jackson.annotation.JsonTypeName;
    +import com.google.common.collect.Lists;
    +
    +@JsonTypeName("table")
    +public class HiveTableWrapper {
    +
    +  @JsonIgnore
    +  private HiveTable table;
    +
    +  @JsonProperty
    +  public String tableName;
    +  @JsonProperty
    +  public String dbName;
    +  @JsonProperty
    +  public String owner;
    +  @JsonProperty
    +  public int createTime;
    +  @JsonProperty
    +  public int lastAccessTime;
    +  @JsonProperty
    +  public int retention;
    +  @JsonProperty
    +  public StorageDescriptorWrapper sd;
    +  @JsonProperty
    +  public List<FieldSchemaWrapper> partitionKeys;
    +  @JsonProperty
    +  public Map<String,String> parameters;
    +  @JsonProperty
    +  public String viewOriginalText;
    +  @JsonProperty
    +  public String viewExpandedText;
    +  @JsonProperty
    +  public String tableType;
    +  @JsonProperty
    +  public ColumnsCacheWrapper columnsCache;
    +
    +  @JsonIgnore
    +  public final Map<String, String> partitionNameTypeMap = new HashMap<>();
    +
    +  @JsonCreator
    +  public HiveTableWrapper(@JsonProperty("tableName") String tableName, @JsonProperty("dbName") String dbName, @JsonProperty("owner") String owner,
    +                          @JsonProperty("createTime") int createTime, @JsonProperty("lastAccessTime") int lastAccessTime,
    +                          @JsonProperty("retention") int retention, @JsonProperty("sd") StorageDescriptorWrapper sd,
    +                          @JsonProperty("partitionKeys") List<FieldSchemaWrapper> partitionKeys, @JsonProperty("parameters") Map<String, String> parameters,
    +                          @JsonProperty("viewOriginalText") String viewOriginalText, @JsonProperty("viewExpandedText") String viewExpandedText,
    +                          @JsonProperty("tableType") String tableType, @JsonProperty("columnsCache") ColumnsCacheWrapper columnsCache
    +  ) {
    +    this.tableName = tableName;
    +    this.dbName = dbName;
    +    this.owner = owner;
    +    this.createTime = createTime;
    +    this.lastAccessTime = lastAccessTime;
    +    this.retention = retention;
    +    this.sd = sd;
    +    this.partitionKeys = partitionKeys;
    +    this.parameters = parameters;
    +    this.viewOriginalText = viewOriginalText;
    +    this.viewExpandedText = viewExpandedText;
    +    this.tableType = tableType;
    +    this.columnsCache = columnsCache;
    +
    +    List<FieldSchema> partitionKeysUnwrapped = Lists.newArrayList();
    +    for (FieldSchemaWrapper w : partitionKeys) {
    +      partitionKeysUnwrapped.add(w.getFieldSchema());
    +      partitionNameTypeMap.put(w.name, w.type);
    +    }
    +    StorageDescriptor sdUnwrapped = sd.getSd();
    +    this.table = new HiveTable(tableName, dbName, owner, createTime, lastAccessTime, retention, sdUnwrapped, partitionKeysUnwrapped,
    +        parameters, viewOriginalText, viewExpandedText, tableType, columnsCache.getColumnListsCache());
    +  }
    +
    +  public HiveTableWrapper(HiveTable table) {
    +    if (table == null) {
    +      return;
    +    }
    +    this.table = table;
    +    this.tableName = table.getTableName();
    +    this.dbName = table.getDbName();
    +    this.owner = table.getOwner();
    +    this.createTime = table.getCreateTime();
    +    this.lastAccessTime = table.getLastAccessTime();
    +    this.retention = table.getRetention();
    +    this.sd = new StorageDescriptorWrapper(table.getSd());
    +    this.partitionKeys = Lists.newArrayList();
    +    for (FieldSchema f : table.getPartitionKeys()) {
    +      this.partitionKeys.add(new FieldSchemaWrapper(f));
    +      partitionNameTypeMap.put(f.getName(), f.getType());
    +    }
    +    this.parameters = table.getParameters();
    +    this.viewOriginalText = table.getViewOriginalText();
    +    this.viewExpandedText = table.getViewExpandedText();
    +    this.tableType = table.getTableType();
    +    this.columnsCache = new ColumnsCacheWrapper(table.getColumnListsCache());
    +  }
    +
    +  @JsonIgnore
    +  public HiveTable getTable() {
    +    return table;
    +  }
    +
    +  @Override
    +  public String toString() {
    +    StringBuilder sb = new StringBuilder("Table(");
    +
    +    sb.append("dbName:");
    +    sb.append(this.dbName);
    +    sb.append(", ");
    +
    +    sb.append("tableName:");
    +    sb.append(this.tableName);
    +    sb.append(")");
    +
    +    return sb.toString();
    +  }
    +
    +  /**
    +   * Wrapper for {@link Partition} class. Used for serialization and deserialization of {@link HivePartition}.
    +   */
    +  public static class HivePartitionWrapper {
    +
    +    @JsonIgnore
    +    private HivePartition partition;
    +
    +    @JsonProperty
    +    public List<String> values;
    +
    +    @JsonProperty
    +    public String tableName;
    +
    +    @JsonProperty
    +    public String dbName;
    +
    +    @JsonProperty
    +    public int createTime;
    +
    +    @JsonProperty
    +    public int lastAccessTime;
    +
    +    @JsonProperty
    +    public StorageDescriptorWrapper sd;
    +
    +    @JsonProperty
    +    public Map<String, String> parameters;
    +
    +    @JsonProperty
    +    private int columnListIndex;
    +
    +    @JsonCreator
    +    public HivePartitionWrapper(@JsonProperty("values") List<String> values, @JsonProperty("tableName") String tableName,
    +                                @JsonProperty("dbName") String dbName, @JsonProperty("createTime") int createTime,
    +                                @JsonProperty("lastAccessTime") int lastAccessTime, @JsonProperty("sd") StorageDescriptorWrapper sd,
    +                                @JsonProperty("parameters") Map<String, String> parameters, @JsonProperty("columnListIndex") int columnListIndex) {
    +      this.values = values;
    +      this.tableName = tableName;
    +      this.dbName = dbName;
    +      this.createTime = createTime;
    +      this.lastAccessTime = lastAccessTime;
    +      this.sd = sd;
    +      this.parameters = parameters;
    +      this.columnListIndex = columnListIndex;
    +
    +      StorageDescriptor sdUnwrapped = sd.getSd();
    +      this.partition = new HivePartition(values, tableName, dbName, createTime, lastAccessTime, sdUnwrapped, parameters, columnListIndex);
    +    }
    +
    +    public HivePartitionWrapper(HivePartition partition) {
    +      if (partition == null) {
    +        return;
    +      }
    +      this.partition = partition;
    +      this.values = partition.getValues();
    +      this.tableName = partition.getTableName();
    +      this.dbName = partition.getDbName();
    +      this.createTime = partition.getCreateTime();
    +      this.lastAccessTime = partition.getLastAccessTime();
    +      this.sd = new StorageDescriptorWrapper(partition.getSd());
    +      this.parameters = partition.getParameters();
    +      this.columnListIndex = partition.getColumnListIndex();
    +    }
    +
    +    @JsonIgnore
    +    public StorageDescriptorWrapper getSd() {
    +      return sd;
    +    }
    +
    +    @JsonIgnore
    +    public HivePartition getPartition() {
    +      return partition;
    +    }
    +
    +    @Override
    +    public String toString() {
    +      StringBuilder sb = new StringBuilder("Partition(");
    +      sb.append("values:");
    +      sb.append(this.values);
    +      sb.append(")");
    +      return sb.toString();
    +    }
    +  }
    +
    +  /**
    +   * Wrapper for {@link StorageDescriptor} class.
    +   * Used in {@link HivePartitionWrapper} and {@link HiveTableWrapper}
    +   * for serialization and deserialization of {@link StorageDescriptor}.
    +   */
    +  public static class StorageDescriptorWrapper {
    +
    +    @JsonIgnore
    +    private StorageDescriptor sd;
    +
    +    // column lists stored in ColumnListsCache
    +    @JsonIgnore
    +    public List<FieldSchemaWrapper> columns;
    +
    +    @JsonProperty
    +    public String location;
    +
    +    @JsonProperty
    +    public String inputFormat;
    +
    +    @JsonProperty
    +    public String outputFormat;
    +
    +    @JsonProperty
    +    public boolean compressed;
    +
    +    @JsonProperty
    +    public int numBuckets;
    +
    +    @JsonProperty
    +    public SerDeInfoWrapper serDeInfo;
    +
    +    @JsonProperty
    +    public List<OrderWrapper> sortCols;
    +
    +    @JsonProperty
    +    public Map<String, String> parameters;
    +
    +    @JsonCreator
    +    public StorageDescriptorWrapper(@JsonProperty("columns") List<FieldSchemaWrapper> columns, @JsonProperty("location") String location, @JsonProperty("inputFormat") String inputFormat,
    +                                    @JsonProperty("outputFormat") String outputFormat, @JsonProperty("compressed") boolean compressed, @JsonProperty("numBuckets") int numBuckets,
    +                                    @JsonProperty("serDeInfo") SerDeInfoWrapper serDeInfo,  @JsonProperty("sortCols") List<OrderWrapper> sortCols,
    +                                    @JsonProperty("parameters") Map<String,String> parameters) {
    +      this.columns = columns;
    +      this.location = location;
    +      this.inputFormat = inputFormat;
    +      this.outputFormat = outputFormat;
    +      this.compressed = compressed;
    +      this.numBuckets = numBuckets;
    +      this.serDeInfo = serDeInfo;
    +      this.sortCols = sortCols;
    +      this.parameters = parameters;
    +      List<FieldSchema> colsUnwrapped;
    +      if (columns != null) {
    +        colsUnwrapped = Lists.newArrayList();
    +        for (FieldSchemaWrapper fieldSchema : columns) {
    +          colsUnwrapped.add(fieldSchema.getFieldSchema());
    +        }
    +      } else {
    +        colsUnwrapped = null;
    +      }
    +      SerDeInfo serDeInfoUnwrapped = serDeInfo.getSerDeInfo();
    +      List<Order> sortColsUnwrapped = Lists.newArrayList();
    +      for (OrderWrapper order : sortCols) {
    +        sortColsUnwrapped.add(order.getOrder());
    +      }
    +      sd = new StorageDescriptor(colsUnwrapped, location, inputFormat, outputFormat,
    +        compressed, numBuckets, serDeInfoUnwrapped, null, sortColsUnwrapped, parameters);
    +    }
    +
    +    public StorageDescriptorWrapper(StorageDescriptor storageDescriptor) {
    +      sd = storageDescriptor;
    +      location = storageDescriptor.getLocation();
    +      inputFormat = storageDescriptor.getInputFormat();
    +      outputFormat = storageDescriptor.getOutputFormat();
    +      compressed = storageDescriptor.isCompressed();
    +      numBuckets = storageDescriptor.getNumBuckets();
    +      serDeInfo = new SerDeInfoWrapper(storageDescriptor.getSerdeInfo());
    +      sortCols = Lists.newArrayList();
    +      for (Order order : storageDescriptor.getSortCols()) {
    +        sortCols.add(new OrderWrapper(order));
    +      }
    +      parameters = storageDescriptor.getParameters();
    +      if (sd.getCols() != null) {
    +        this.columns = Lists.newArrayList();
    +        for (FieldSchema fieldSchema : sd.getCols()) {
    +          this.columns.add(new FieldSchemaWrapper(fieldSchema));
    +        }
    +      }
    +    }
    +
    +    @JsonIgnore
    +    public StorageDescriptor getSd() {
    +      return sd;
    +    }
    +
    +    @JsonIgnore
    +    public void setColumns(List<FieldSchema> columns) {
    +      sd.setCols(columns);
    +      if (columns != null) {
    +        this.columns = Lists.newArrayList();
    +        for (FieldSchema fieldSchema : columns) {
    +          this.columns.add(new FieldSchemaWrapper(fieldSchema));
    +        }
    +      } else {
    +        this.columns = null;
    +      }
    +    }
    +
    +    @JsonIgnore
    +    public List<FieldSchemaWrapper> getColumns() {
    +      return columns;
    +    }
    +  }
    +
    +  public static class SerDeInfoWrapper {
    --- End diff --
    
    Any particular reason why you moved those wrapper class around? It makes the code change much larger, and more difficult to understand the real change in the patch. 
    

---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---