parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tians...@apache.org
Subject git commit: PARQUET-94: Fix bug in ParquetScroogeScheme constructor, minor cleanup
Date Mon, 22 Sep 2014 18:11:18 GMT
Repository: incubator-parquet-mr
Updated Branches:
  refs/heads/master 501e8feac -> 9cdcf3bbd


PARQUET-94: Fix bug in ParquetScroogeScheme constructor, minor cleanup

I noticed that ParquetScroogeScheme's constructor ignores the provided klass argument.
I also added in missing type parameters for the Config object where they were missing.

Author: Alex Levenson <alexlevenson@twitter.com>

Closes #61 from isnotinvain/alexlevenson/parquet-scrooge-cleanup and squashes the following
commits:

2b16007 [Alex Levenson] Fix bug in ParquetScroogeScheme constructor, minor cleanup


Project: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/commit/9cdcf3bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/tree/9cdcf3bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/diff/9cdcf3bb

Branch: refs/heads/master
Commit: 9cdcf3bbdf8f772d3fadf388b2db048598c155e9
Parents: 501e8fe
Author: Alex Levenson <alexlevenson@twitter.com>
Authored: Mon Sep 22 11:11:08 2014 -0700
Committer: Tianshuo Deng <tdeng@twitter.com>
Committed: Mon Sep 22 11:11:08 2014 -0700

----------------------------------------------------------------------
 .../parquet/cascading/ParquetTBaseScheme.java   | 13 +++++------
 .../parquet/cascading/ParquetValueScheme.java   | 23 ++++++++++----------
 .../parquet/scrooge/ParquetScroogeScheme.java   | 10 +++------
 3 files changed, 19 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/9cdcf3bb/parquet-cascading/src/main/java/parquet/cascading/ParquetTBaseScheme.java
----------------------------------------------------------------------
diff --git a/parquet-cascading/src/main/java/parquet/cascading/ParquetTBaseScheme.java b/parquet-cascading/src/main/java/parquet/cascading/ParquetTBaseScheme.java
index 40817af..c84addc 100644
--- a/parquet-cascading/src/main/java/parquet/cascading/ParquetTBaseScheme.java
+++ b/parquet-cascading/src/main/java/parquet/cascading/ParquetTBaseScheme.java
@@ -26,7 +26,6 @@ import parquet.filter2.predicate.FilterPredicate;
 import parquet.hadoop.ParquetInputFormat;
 import parquet.hadoop.mapred.DeprecatedParquetInputFormat;
 import parquet.hadoop.mapred.DeprecatedParquetOutputFormat;
-import parquet.hadoop.thrift.ParquetThriftInputFormat;
 import parquet.hadoop.thrift.ThriftReadSupport;
 import parquet.hadoop.thrift.ThriftWriteSupport;
 import parquet.thrift.TBaseRecordConverter;
@@ -35,26 +34,25 @@ public class ParquetTBaseScheme<T extends TBase<?,?>> extends
ParquetValueScheme
 
   // In the case of reads, we can read the thrift class from the file metadata
   public ParquetTBaseScheme() {
-    this(new Config());
+    this(new Config<T>());
   }
 
   public ParquetTBaseScheme(Class<T> thriftClass) {
-    this(new Config().withRecordClass(thriftClass));
+    this(new Config<T>().withRecordClass(thriftClass));
   }
 
   public ParquetTBaseScheme(FilterPredicate filterPredicate) {
-    this(new Config().withFilterPredicate(filterPredicate));
+    this(new Config<T>().withFilterPredicate(filterPredicate));
   }
 
   public ParquetTBaseScheme(FilterPredicate filterPredicate, Class<T> thriftClass)
{
-    this(new Config().withRecordClass(thriftClass).withFilterPredicate(filterPredicate));
+    this(new Config<T>().withRecordClass(thriftClass).withFilterPredicate(filterPredicate));
   }
 
-  public ParquetTBaseScheme(Config config) {
+  public ParquetTBaseScheme(Config<T> config) {
     super(config);
   }
 
-  @SuppressWarnings("rawtypes")
   @Override
   public void sourceConfInit(FlowProcess<JobConf> fp,
       Tap<JobConf, RecordReader, OutputCollector> tap, JobConf jobConf) {
@@ -64,7 +62,6 @@ public class ParquetTBaseScheme<T extends TBase<?,?>> extends
ParquetValueScheme
     ThriftReadSupport.setRecordConverterClass(jobConf, TBaseRecordConverter.class);
   }
 
-  @SuppressWarnings("rawtypes")
   @Override
   public void sinkConfInit(FlowProcess<JobConf> fp,
       Tap<JobConf, RecordReader, OutputCollector> tap, JobConf jobConf) {

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/9cdcf3bb/parquet-cascading/src/main/java/parquet/cascading/ParquetValueScheme.java
----------------------------------------------------------------------
diff --git a/parquet-cascading/src/main/java/parquet/cascading/ParquetValueScheme.java b/parquet-cascading/src/main/java/parquet/cascading/ParquetValueScheme.java
index 6e8c13a..b994ff6 100644
--- a/parquet-cascading/src/main/java/parquet/cascading/ParquetValueScheme.java
+++ b/parquet-cascading/src/main/java/parquet/cascading/ParquetValueScheme.java
@@ -32,10 +32,8 @@ import cascading.tuple.TupleEntry;
 import parquet.filter2.predicate.FilterPredicate;
 import parquet.hadoop.ParquetInputFormat;
 import parquet.hadoop.mapred.Container;
-import parquet.hadoop.mapred.DeprecatedParquetOutputFormat;
 import parquet.hadoop.thrift.ParquetThriftInputFormat;
 import parquet.hadoop.thrift.ThriftReadSupport;
-import parquet.hadoop.thrift.ThriftWriteSupport;
 
 import static parquet.Preconditions.checkNotNull;
 
@@ -76,31 +74,31 @@ public abstract class ParquetValueScheme<T> extends Scheme<JobConf,
RecordReader
       return klass;
     }
 
-    public Config withFilterPredicate(FilterPredicate f) {
-      return new Config(this.klass, checkNotNull(f, "filterPredicate"), this.projectionString);
+    public Config<T> withFilterPredicate(FilterPredicate f) {
+      return new Config<T>(this.klass, checkNotNull(f, "filterPredicate"), this.projectionString);
     }
 
-    public Config withProjectionString(String p) {
-      return new Config(this.klass, this.filterPredicate, checkNotNull(p, "projectionFilter"));
+    public Config<T> withProjectionString(String p) {
+      return new Config<T>(this.klass, this.filterPredicate, checkNotNull(p, "projectionFilter"));
     }
 
-    public Config withRecordClass(Class<T> klass) {
-      return new Config(checkNotNull(klass, "recordClass"), this.filterPredicate, this.projectionString);
+    public Config<T> withRecordClass(Class<T> klass) {
+      return new Config<T>(checkNotNull(klass, "recordClass"), this.filterPredicate,
this.projectionString);
     }
   }
 
   private static final long serialVersionUID = 157560846420730043L;
-  protected final Config config;
+  protected final Config<T> config;
 
   public ParquetValueScheme() {
-    this(new Config());
+    this(new Config<T>());
   }
 
   public ParquetValueScheme(FilterPredicate filterPredicate) {
-    this(new Config().withFilterPredicate(filterPredicate));
+    this(new Config<T>().withFilterPredicate(filterPredicate));
   }
 
-  public ParquetValueScheme(Config config) {
+  public ParquetValueScheme(Config<T> config) {
     this.config = config;
   }
 
@@ -143,6 +141,7 @@ public abstract class ParquetValueScheme<T> extends Scheme<JobConf,
RecordReader
     return true;
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   public void sink(FlowProcess<JobConf> fp, SinkCall<Object[], OutputCollector>
sc)
       throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/9cdcf3bb/parquet-scrooge/src/main/java/parquet/scrooge/ParquetScroogeScheme.java
----------------------------------------------------------------------
diff --git a/parquet-scrooge/src/main/java/parquet/scrooge/ParquetScroogeScheme.java b/parquet-scrooge/src/main/java/parquet/scrooge/ParquetScroogeScheme.java
index 0f46f8f..1ca9cc6 100644
--- a/parquet-scrooge/src/main/java/parquet/scrooge/ParquetScroogeScheme.java
+++ b/parquet-scrooge/src/main/java/parquet/scrooge/ParquetScroogeScheme.java
@@ -31,7 +31,6 @@ import parquet.cascading.ParquetValueScheme;
 import parquet.filter2.predicate.FilterPredicate;
 import parquet.hadoop.ParquetInputFormat;
 import parquet.hadoop.mapred.DeprecatedParquetInputFormat;
-import parquet.hadoop.thrift.ParquetThriftInputFormat;
 import parquet.hadoop.thrift.ThriftReadSupport;
 
 public class ParquetScroogeScheme<T extends ThriftStruct> extends ParquetValueScheme<T>
{
@@ -39,18 +38,17 @@ public class ParquetScroogeScheme<T extends ThriftStruct> extends
ParquetValueSc
   private static final long serialVersionUID = -8332274507341448397L;
 
   public ParquetScroogeScheme(Class<T> klass) {
-    this(new Config().withRecordClass(klass));
+    this(new Config<T>().withRecordClass(klass));
   }
 
   public ParquetScroogeScheme(FilterPredicate filterPredicate, Class<T> klass) {
-    this(new Config().withFilterPredicate(filterPredicate));
+    this(new Config<T>().withFilterPredicate(filterPredicate).withRecordClass(klass));
   }
 
-  public ParquetScroogeScheme(Config config) {
+  public ParquetScroogeScheme(Config<T> config) {
     super(config);
   }
 
-  @SuppressWarnings("rawtypes")
   @Override
   public void sinkConfInit(FlowProcess<JobConf> arg0,
       Tap<JobConf, RecordReader, OutputCollector> arg1, JobConf arg2) {
@@ -63,8 +61,6 @@ public class ParquetScroogeScheme<T extends ThriftStruct> extends
ParquetValueSc
   @Override
   public boolean isSink() { return false; }
 
-
-  @SuppressWarnings("rawtypes")
   @Override
   public void sourceConfInit(FlowProcess<JobConf> fp,
       Tap<JobConf, RecordReader, OutputCollector> tap, JobConf jobConf) {


Mime
View raw message