drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ve...@apache.org
Subject [3/5] drill git commit: DRILL-2367: Removed partition column label in conf (gets from options)
Date Thu, 26 Mar 2015 15:59:18 GMT
DRILL-2367: Removed partition column label in conf (gets from options)


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/06d7eb28
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/06d7eb28
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/06d7eb28

Branch: refs/heads/master
Commit: 06d7eb28e75d72ccb7d1c755a43b891212353221
Parents: c5ec780
Author: Sudheesh Katkam <skatkam@maprtech.com>
Authored: Wed Mar 18 16:48:36 2015 -0700
Committer: Sudheesh Katkam <skatkam@maprtech.com>
Committed: Wed Mar 25 18:16:30 2015 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/drill/exec/ExecConstants.java  | 8 ++++++++
 .../java/org/apache/drill/exec/physical/impl/ScanBatch.java | 9 +++++++--
 .../apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java  | 3 ++-
 .../drill/exec/store/parquet/ParquetScanBatchCreator.java   | 3 ++-
 exec/java-exec/src/main/resources/drill-module.conf         | 3 +--
 5 files changed, 20 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/06d7eb28/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index cd0a0a2..14e6ad1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -105,8 +105,16 @@ public interface ExecConstants {
   public static String JSON_ALL_TEXT_MODE = "store.json.all_text_mode";
   public static OptionValidator JSON_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(JSON_ALL_TEXT_MODE,
false);
 
+  /**
+   * The column label (for directory levels) in results when querying files in a directory
+   * E.g.  labels: dir0   dir1
+   *    structure: foo
+   *                |-    bar  -  a.parquet
+   *                |-    baz  -  b.parquet
+   */
   public static final String FILESYSTEM_PARTITION_COLUMN_LABEL = "drill.exec.storage.file.partition.column.label";
   public static final OptionValidator FILESYSTEM_PARTITION_COLUMN_LABEL_VALIDATOR = new StringValidator(FILESYSTEM_PARTITION_COLUMN_LABEL,
"dir");
+
   public static String MONGO_ALL_TEXT_MODE = "store.mongo.all_text_mode";
   public static OptionValidator MONGO_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(MONGO_ALL_TEXT_MODE,
false);
 

http://git-wip-us.apache.org/repos/asf/drill/blob/06d7eb28/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index 6b7294d..ca2a048 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -46,6 +46,7 @@ import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.record.WritableBatch;
 import org.apache.drill.exec.record.selection.SelectionVector2;
 import org.apache.drill.exec.record.selection.SelectionVector4;
+import org.apache.drill.exec.server.options.OptionValue;
 import org.apache.drill.exec.store.RecordReader;
 import org.apache.drill.exec.vector.AllocationHelper;
 import org.apache.drill.exec.vector.NullableVarCharVector;
@@ -103,8 +104,12 @@ public class ScanBatch implements RecordBatch {
     this.partitionColumns = partitionColumns.iterator();
     this.partitionValues = this.partitionColumns.hasNext() ? this.partitionColumns.next()
: null;
     this.selectedPartitionColumns = selectedPartitionColumns;
-    DrillConfig config = context.getConfig();
-    this.partitionColumnDesignator = config == null ? "dir" : config.getString(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL);
+
+    // TODO Remove null check after DRILL-2097 is resolved. That JIRA refers to test cases
that do not initialize
+    // options; so labelValue = null.
+    final OptionValue labelValue = context.getOptions().getOption(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL);
+    this.partitionColumnDesignator = labelValue == null ? "dir" : labelValue.string_val;
+
     addPartitionVectors();
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/06d7eb28/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
index 431b362..6e1e0cc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
@@ -118,7 +118,8 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig>
implements
       List<SchemaPath> columns) throws ExecutionSetupException;
 
   RecordBatch getReaderBatch(FragmentContext context, EasySubScan scan) throws ExecutionSetupException
{
-    String partitionDesignator = context.getConfig().getString(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL);
+    String partitionDesignator = context.getOptions()
+      .getOption(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL).string_val;
     List<SchemaPath> columns = scan.getColumns();
     List<RecordReader> readers = Lists.newArrayList();
     List<String[]> partitionColumns = Lists.newArrayList();

http://git-wip-us.apache.org/repos/asf/drill/blob/06d7eb28/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index 3ae2b36..c1f815e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -63,7 +63,8 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
   @Override
   public RecordBatch getBatch(FragmentContext context, ParquetRowGroupScan rowGroupScan,
List<RecordBatch> children) throws ExecutionSetupException {
     Preconditions.checkArgument(children.isEmpty());
-    String partitionDesignator = context.getConfig().getString(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL);
+    String partitionDesignator = context.getOptions()
+      .getOption(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL).string_val;
     List<SchemaPath> columns = rowGroupScan.getColumns();
     List<RecordReader> readers = Lists.newArrayList();
     OperatorContext oContext = new OperatorContext(rowGroupScan, context,

http://git-wip-us.apache.org/repos/asf/drill/blob/06d7eb28/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index 9e807b0..af225c4 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -60,8 +60,7 @@ drill.exec: {
       text: {
         buffer.size: 262144,
         batch.size: 4000
-      },
-      partition.column.label: "dir"
+      }
     }
   },
   metrics : {


Mime
View raw message