drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] [drill] amansinha100 commented on a change in pull request #1771: DRILL-7199: Optimize population of metadata for non-interesting columns
Date Wed, 01 May 2019 15:58:14 GMT
amansinha100 commented on a change in pull request #1771: DRILL-7199: Optimize population of
metadata for non-interesting columns
URL: https://github.com/apache/drill/pull/1771#discussion_r280106679
 
 

 ##########
 File path: exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java
 ##########
 @@ -286,25 +287,27 @@ public static PartitionMetadata getPartitionMetadata(SchemaPath partitionColumn,
       statistics.put(ColumnStatisticsKind.NULLS_COUNT, nulls);
       columnsStatistics.put(colPath, new ColumnStatisticsImpl(statistics, comparator));
     }
-    columnsStatistics.putAll(populateNonInterestingColumnsStats(columnsStatistics.keySet(),
tableMetadata));
     return columnsStatistics;
   }
 
   /**
-   * Populates the non-interesting column's statistics
-   * @param schemaPaths columns paths which should be ignored
+   * Returns the non-interesting column's metadata
    * @param parquetTableMetadata the source of column metadata for non-interesting column's
statistics
-   * @return returns non-interesting column statistics map
+   * @return returns non-interesting columns metadata
    */
-  @SuppressWarnings("unchecked")
-  public static Map<SchemaPath, ColumnStatistics> populateNonInterestingColumnsStats(
-          Set<SchemaPath> schemaPaths, MetadataBase.ParquetTableMetadataBase parquetTableMetadata)
{
+  public static NonInterestingColumnsMetadata getNonInterestingColumnsMeta(MetadataBase.ParquetTableMetadataBase
parquetTableMetadata) {
     Map<SchemaPath, ColumnStatistics> columnsStatistics = new HashMap<>();
     if (parquetTableMetadata instanceof Metadata_V4.ParquetTableMetadata_v4) {
-      for (Metadata_V4.ColumnTypeMetadata_v4 columnTypeMetadata :
-          ((Metadata_V4.ParquetTableMetadata_v4) parquetTableMetadata).getColumnTypeInfoMap().values())
{
-        SchemaPath schemaPath = SchemaPath.getCompoundPath(columnTypeMetadata.name);
-        if (!schemaPaths.contains(schemaPath)) {
+      ConcurrentHashMap<Metadata_V4.ColumnTypeMetadata_v4.Key, Metadata_V4.ColumnTypeMetadata_v4>
columnTypeInfoMap =
+              ((Metadata_V4.ParquetTableMetadata_v4) parquetTableMetadata).getColumnTypeInfoMap();
+
+      if (columnTypeInfoMap == null) {
+        return new NonInterestingColumnsMetadata(columnsStatistics);
+      } // in some cases for runtime pruning
 
 Review comment:
   It's not clear what this comment means ..

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message