drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject [12/12] drill git commit: DRILL-5229: Update kudu-client to 1.3.0
Date Sat, 03 Jun 2017 04:46:07 GMT
DRILL-5229: Update kudu-client to 1.3.0

closes #828


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/dd2692ec
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/dd2692ec
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/dd2692ec

Branch: refs/heads/master
Commit: dd2692ecd57a180f33e51147afe063627552979e
Parents: 676ea88
Author: eskabetxe <boto@boto.pro>
Authored: Sat May 6 13:41:36 2017 +0200
Committer: Jinfeng Ni <jni@apache.org>
Committed: Fri Jun 2 21:43:14 2017 -0700

----------------------------------------------------------------------
 contrib/storage-kudu/pom.xml                    |  5 ++-
 .../codegen/templates/KuduRecordWriter.java     |  4 +--
 .../drill/exec/store/kudu/DrillKuduTable.java   | 14 ++++----
 .../drill/exec/store/kudu/KuduGroupScan.java    |  4 +--
 .../drill/exec/store/kudu/KuduRecordReader.java | 30 ++++++++--------
 .../exec/store/kudu/KuduRecordWriterImpl.java   | 37 ++++++++++----------
 .../exec/store/kudu/KuduSchemaFactory.java      |  6 ++--
 .../exec/store/kudu/KuduStoragePlugin.java      |  2 +-
 .../drill/store/kudu/TestKuduConnect.java       | 30 ++++++++--------
 9 files changed, 67 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/pom.xml b/contrib/storage-kudu/pom.xml
index 2ba9cac..74e6eb8 100644
--- a/contrib/storage-kudu/pom.xml
+++ b/contrib/storage-kudu/pom.xml
@@ -18,7 +18,6 @@
   </parent>
 
   <artifactId>drill-kudu-storage</artifactId>
-
   <name>contrib/kudu-storage-plugin</name>
 
 
@@ -47,9 +46,9 @@
     </dependency>
 
     <dependency>
-      <groupId>org.kududb</groupId>
+      <groupId>org.apache.kudu</groupId>
       <artifactId>kudu-client</artifactId>
-      <version>0.6.0</version>
+      <version>1.3.0</version>
     </dependency>
 
   </dependencies>

http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/codegen/templates/KuduRecordWriter.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/codegen/templates/KuduRecordWriter.java b/contrib/storage-kudu/src/main/codegen/templates/KuduRecordWriter.java
index 01c7c28..2b76cac 100644
--- a/contrib/storage-kudu/src/main/codegen/templates/KuduRecordWriter.java
+++ b/contrib/storage-kudu/src/main/codegen/templates/KuduRecordWriter.java
@@ -88,7 +88,7 @@ import java.lang.UnsupportedOperationException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import org.kududb.client.*;
+import org.apache.kudu.client.*;
 import org.apache.drill.exec.store.*;
 
 public abstract class KuduRecordWriter extends AbstractRecordWriter implements RecordWriter
{
@@ -157,7 +157,7 @@ public abstract class KuduRecordWriter extends AbstractRecordWriter implements
R
             <#elseif minor.class == "VarChar" >
               byte[] bytes = new byte[holder.end - holder.start];
               holder.buffer.getBytes(holder.start, bytes);
-              row.addStringUtf8(fieldId, bytes);
+              row.addString(fieldId, new String(bytes));
             <#elseif minor.class == "VarBinary">
               byte[] bytes = new byte[holder.end - holder.start];
               holder.buffer.getBytes(holder.start, bytes);

http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/DrillKuduTable.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/DrillKuduTable.java
b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/DrillKuduTable.java
index 3fc69c6..8404aac 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/DrillKuduTable.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/DrillKuduTable.java
@@ -23,9 +23,9 @@ import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.drill.exec.planner.logical.DynamicDrillTable;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
 
 import com.google.common.collect.Lists;
 
@@ -56,8 +56,6 @@ public class DrillKuduTable extends DynamicDrillTable {
 
   private RelDataType getSqlTypeFromKuduType(RelDataTypeFactory typeFactory, Type type) {
     switch (type) {
-    case BINARY:
-      return typeFactory.createSqlType(SqlTypeName.VARBINARY, Integer.MAX_VALUE);
     case BOOL:
       return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
     case DOUBLE:
@@ -70,9 +68,11 @@ public class DrillKuduTable extends DynamicDrillTable {
     case INT8:
       return typeFactory.createSqlType(SqlTypeName.INTEGER);
     case STRING:
-      return typeFactory.createSqlType(SqlTypeName.VARCHAR, Integer.MAX_VALUE);
-    case TIMESTAMP:
+      return typeFactory.createSqlType(SqlTypeName.VARCHAR);
+    case UNIXTIME_MICROS:
       return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
+    case BINARY:
+      return typeFactory.createSqlType(SqlTypeName.VARBINARY, Integer.MAX_VALUE);
     default:
       throw new UnsupportedOperationException("Unsupported type.");
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduGroupScan.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduGroupScan.java
b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduGroupScan.java
index 873f216..dfc3c44 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduGroupScan.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduGroupScan.java
@@ -51,8 +51,8 @@ import org.apache.drill.exec.store.schedule.AssignmentCreator;
 import org.apache.drill.exec.store.schedule.CompleteWork;
 import org.apache.drill.exec.store.schedule.EndpointByteMap;
 import org.apache.drill.exec.store.schedule.EndpointByteMapImpl;
-import org.kududb.client.LocatedTablet;
-import org.kududb.client.LocatedTablet.Replica;
+import org.apache.kudu.client.LocatedTablet;
+import org.apache.kudu.client.LocatedTablet.Replica;
 
 @JsonTypeName("kudu-scan")
 public class KuduGroupScan extends AbstractGroupScan {

http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
index 541daa4..ef7efcf 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordReader.java
@@ -52,16 +52,16 @@ import org.apache.drill.exec.vector.TimeStampVector;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.VarBinaryVector;
 import org.apache.drill.exec.vector.VarCharVector;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.KuduClient;
-import org.kududb.client.KuduScanner;
-import org.kududb.client.KuduScanner.KuduScannerBuilder;
-import org.kududb.client.KuduTable;
-import org.kududb.client.RowResult;
-import org.kududb.client.RowResultIterator;
-import org.kududb.client.shaded.com.google.common.collect.ImmutableMap;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduScanner;
+import org.apache.kudu.client.KuduScanner.KuduScannerBuilder;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.RowResult;
+import org.apache.kudu.client.RowResultIterator;
+import org.apache.kudu.client.shaded.com.google.common.collect.ImmutableMap;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -114,8 +114,8 @@ public class KuduRecordReader extends AbstractRecordReader {
       context.getStats().startWait();
       try {
         scanner = builder
-            .lowerBoundPartitionKeyRaw(scanSpec.getStartKey())
-            .exclusiveUpperBoundPartitionKeyRaw(scanSpec.getEndKey())
+            .lowerBoundRaw(scanSpec.getStartKey())
+            .exclusiveUpperBoundRaw(scanSpec.getEndKey())
             .build();
       } finally {
         context.getStats().stopWait();
@@ -138,7 +138,7 @@ public class KuduRecordReader extends AbstractRecordReader {
         .put(Type.INT32, MinorType.INT)
         .put(Type.INT64, MinorType.BIGINT)
         .put(Type.STRING, MinorType.VARCHAR)
-        .put(Type.TIMESTAMP, MinorType.TIMESTAMP)
+        .put(Type.UNIXTIME_MICROS, MinorType.TIMESTAMP)
         .build();
   }
 
@@ -236,7 +236,7 @@ public class KuduRecordReader extends AbstractRecordReader {
         break;
       }
       case STRING: {
-        ByteBuffer value = result.getBinary(pci.index);
+        ByteBuffer value = ByteBuffer.wrap(result.getString(pci.index).getBytes());
         if (pci.kuduColumn.isNullable()) {
           ((NullableVarCharVector.Mutator) pci.vv.getMutator())
               .setSafe(rowIndex, value, 0, value.remaining());
@@ -309,7 +309,7 @@ public class KuduRecordReader extends AbstractRecordReader {
               .setSafe(rowIndex, result.getLong(pci.index));
         }
         break;
-      case TIMESTAMP:
+        case UNIXTIME_MICROS:
         if (pci.kuduColumn.isNullable()) {
           ((NullableTimeStampVector.Mutator) pci.vv.getMutator())
               .setSafe(rowIndex, result.getLong(pci.index) / 1000);

http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
index 6b39cc5..2e40acf 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduRecordWriterImpl.java
@@ -17,11 +17,6 @@
  */
 package org.apache.drill.exec.store.kudu;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MajorType;
@@ -29,15 +24,21 @@ import org.apache.drill.exec.ops.OperatorContext;
 import org.apache.drill.exec.record.BatchSchema;
 import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.record.VectorAccessible;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.Insert;
-import org.kududb.client.KuduClient;
-import org.kududb.client.KuduSession;
-import org.kududb.client.KuduTable;
-import org.kududb.client.OperationResponse;
-import org.kududb.client.SessionConfiguration.FlushMode;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.Insert;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduSession;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.CreateTableOptions;
+import org.apache.kudu.client.OperationResponse;
+import org.apache.kudu.client.SessionConfiguration.FlushMode;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 
 public class KuduRecordWriterImpl extends KuduRecordWriter {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(KuduRecordWriterImpl.class);
@@ -81,7 +82,7 @@ public class KuduRecordWriterImpl extends KuduRecordWriter {
           i++;
         }
         Schema kuduSchema = new Schema(columns);
-        table = client.createTable(name, kuduSchema);
+        table = client.createTable(name, kuduSchema, new CreateTableOptions());
       }
     } catch (Exception e) {
       throw new IOException(e);
@@ -113,11 +114,11 @@ public class KuduRecordWriterImpl extends KuduRecordWriter {
     case INT:
       return Type.INT32;
     case TIMESTAMP:
-      return Type.TIMESTAMP;
-    case VARBINARY:
-      return Type.BINARY;
+      return Type.UNIXTIME_MICROS;
     case VARCHAR:
       return Type.STRING;
+    case VARBINARY:
+      return Type.BINARY;
     default:
       throw UserException
         .dataWriteError()

http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduSchemaFactory.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduSchemaFactory.java
b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduSchemaFactory.java
index 34e5b2a..4d9caf3 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduSchemaFactory.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduSchemaFactory.java
@@ -31,9 +31,9 @@ import org.apache.drill.exec.planner.logical.CreateTableEntry;
 import org.apache.drill.exec.store.AbstractSchema;
 import org.apache.drill.exec.store.SchemaConfig;
 import org.apache.drill.exec.store.SchemaFactory;
-import org.kududb.Schema;
-import org.kududb.client.KuduTable;
-import org.kududb.client.ListTablesResponse;
+import org.apache.kudu.Schema;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.ListTablesResponse;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Sets;

http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduStoragePlugin.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduStoragePlugin.java
b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduStoragePlugin.java
index 15aa469..0d98755 100644
--- a/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduStoragePlugin.java
+++ b/contrib/storage-kudu/src/main/java/org/apache/drill/exec/store/kudu/KuduStoragePlugin.java
@@ -24,7 +24,7 @@ import org.apache.drill.common.JSONOptions;
 import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.store.AbstractStoragePlugin;
 import org.apache.drill.exec.store.SchemaConfig;
-import org.kududb.client.KuduClient;
+import org.apache.kudu.client.KuduClient;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;

http://git-wip-us.apache.org/repos/asf/drill/blob/dd2692ec/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduConnect.java
----------------------------------------------------------------------
diff --git a/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduConnect.java
b/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduConnect.java
index 0ee0134..2391fc9 100644
--- a/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduConnect.java
+++ b/contrib/storage-kudu/src/test/java/org/apache/drill/store/kudu/TestKuduConnect.java
@@ -18,24 +18,25 @@
 package org.apache.drill.store.kudu;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import org.junit.Ignore;
 import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.CreateTableOptions;
-import org.kududb.client.Insert;
-import org.kududb.client.KuduClient;
-import org.kududb.client.KuduScanner;
-import org.kududb.client.KuduSession;
-import org.kududb.client.KuduTable;
-import org.kududb.client.ListTablesResponse;
-import org.kududb.client.PartialRow;
-import org.kududb.client.RowResult;
-import org.kududb.client.RowResultIterator;
-import org.kududb.client.SessionConfiguration;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.CreateTableOptions;
+import org.apache.kudu.client.Insert;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduScanner;
+import org.apache.kudu.client.KuduSession;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.ListTablesResponse;
+import org.apache.kudu.client.PartialRow;
+import org.apache.kudu.client.RowResult;
+import org.apache.kudu.client.RowResultIterator;
+import org.apache.kudu.client.SessionConfiguration;
 
 @Ignore("requires remote kudu server")
 public class TestKuduConnect {
@@ -63,6 +64,7 @@ public class TestKuduConnect {
 
       CreateTableOptions builder = new CreateTableOptions();
       builder.setNumReplicas(replicas);
+      builder.setRangePartitionColumns(Arrays.asList("key"));
       for (int i = 1; i < tablets; i++) {
         PartialRow splitRow = schema.newPartialRow();
         splitRow.addInt("key", i*1000);


Mime
View raw message