Return-Path: X-Original-To: apmail-drill-commits-archive@www.apache.org Delivered-To: apmail-drill-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 28103171DF for ; Thu, 26 Mar 2015 15:59:51 +0000 (UTC) Received: (qmail 58998 invoked by uid 500); 26 Mar 2015 15:59:16 -0000 Delivered-To: apmail-drill-commits-archive@drill.apache.org Received: (qmail 58940 invoked by uid 500); 26 Mar 2015 15:59:16 -0000 Mailing-List: contact commits-help@drill.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: commits@drill.apache.org Delivered-To: mailing list commits@drill.apache.org Received: (qmail 58859 invoked by uid 99); 26 Mar 2015 15:59:16 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 Mar 2015 15:59:16 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2081BE2F1E; Thu, 26 Mar 2015 15:59:16 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: venki@apache.org To: commits@drill.apache.org Date: Thu, 26 Mar 2015 15:59:18 -0000 Message-Id: In-Reply-To: <9c49ed97a52b4068a40c2e623abb6b56@git.apache.org> References: <9c49ed97a52b4068a40c2e623abb6b56@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/5] drill git commit: DRILL-2367: Removed partition column label in conf (gets from options) DRILL-2367: Removed partition column label in conf (gets from options) Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/06d7eb28 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/06d7eb28 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/06d7eb28 Branch: refs/heads/master Commit: 06d7eb28e75d72ccb7d1c755a43b891212353221 Parents: c5ec780 Author: Sudheesh Katkam Authored: Wed Mar 18 16:48:36 2015 -0700 Committer: Sudheesh Katkam Committed: Wed Mar 25 18:16:30 2015 -0700 ---------------------------------------------------------------------- .../src/main/java/org/apache/drill/exec/ExecConstants.java | 8 ++++++++ .../java/org/apache/drill/exec/physical/impl/ScanBatch.java | 9 +++++++-- .../apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java | 3 ++- .../drill/exec/store/parquet/ParquetScanBatchCreator.java | 3 ++- exec/java-exec/src/main/resources/drill-module.conf | 3 +-- 5 files changed, 20 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/06d7eb28/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index cd0a0a2..14e6ad1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -105,8 +105,16 @@ public interface ExecConstants { public static String JSON_ALL_TEXT_MODE = "store.json.all_text_mode"; public static OptionValidator JSON_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(JSON_ALL_TEXT_MODE, false); + /** + * The column label (for directory levels) in results when querying files in a directory + * E.g. labels: dir0 dir1 + * structure: foo + * |- bar - a.parquet + * |- baz - b.parquet + */ public static final String FILESYSTEM_PARTITION_COLUMN_LABEL = "drill.exec.storage.file.partition.column.label"; public static final OptionValidator FILESYSTEM_PARTITION_COLUMN_LABEL_VALIDATOR = new StringValidator(FILESYSTEM_PARTITION_COLUMN_LABEL, "dir"); + public static String MONGO_ALL_TEXT_MODE = "store.mongo.all_text_mode"; public static OptionValidator MONGO_READER_ALL_TEXT_MODE_VALIDATOR = new BooleanValidator(MONGO_ALL_TEXT_MODE, false); http://git-wip-us.apache.org/repos/asf/drill/blob/06d7eb28/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java index 6b7294d..ca2a048 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java @@ -46,6 +46,7 @@ import org.apache.drill.exec.record.VectorWrapper; import org.apache.drill.exec.record.WritableBatch; import org.apache.drill.exec.record.selection.SelectionVector2; import org.apache.drill.exec.record.selection.SelectionVector4; +import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.vector.AllocationHelper; import org.apache.drill.exec.vector.NullableVarCharVector; @@ -103,8 +104,12 @@ public class ScanBatch implements RecordBatch { this.partitionColumns = partitionColumns.iterator(); this.partitionValues = this.partitionColumns.hasNext() ? this.partitionColumns.next() : null; this.selectedPartitionColumns = selectedPartitionColumns; - DrillConfig config = context.getConfig(); - this.partitionColumnDesignator = config == null ? "dir" : config.getString(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL); + + // TODO Remove null check after DRILL-2097 is resolved. That JIRA refers to test cases that do not initialize + // options; so labelValue = null. + final OptionValue labelValue = context.getOptions().getOption(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL); + this.partitionColumnDesignator = labelValue == null ? "dir" : labelValue.string_val; + addPartitionVectors(); } http://git-wip-us.apache.org/repos/asf/drill/blob/06d7eb28/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java index 431b362..6e1e0cc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java @@ -118,7 +118,8 @@ public abstract class EasyFormatPlugin implements List columns) throws ExecutionSetupException; RecordBatch getReaderBatch(FragmentContext context, EasySubScan scan) throws ExecutionSetupException { - String partitionDesignator = context.getConfig().getString(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL); + String partitionDesignator = context.getOptions() + .getOption(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL).string_val; List columns = scan.getColumns(); List readers = Lists.newArrayList(); List partitionColumns = Lists.newArrayList(); http://git-wip-us.apache.org/repos/asf/drill/blob/06d7eb28/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java index 3ae2b36..c1f815e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java @@ -63,7 +63,8 @@ public class ParquetScanBatchCreator implements BatchCreator children) throws ExecutionSetupException { Preconditions.checkArgument(children.isEmpty()); - String partitionDesignator = context.getConfig().getString(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL); + String partitionDesignator = context.getOptions() + .getOption(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL).string_val; List columns = rowGroupScan.getColumns(); List readers = Lists.newArrayList(); OperatorContext oContext = new OperatorContext(rowGroupScan, context, http://git-wip-us.apache.org/repos/asf/drill/blob/06d7eb28/exec/java-exec/src/main/resources/drill-module.conf ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index 9e807b0..af225c4 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -60,8 +60,7 @@ drill.exec: { text: { buffer.size: 262144, batch.size: 4000 - }, - partition.column.label: "dir" + } } }, metrics : {