hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ych...@apache.org
Subject hive git commit: HIVE-15437: avro tables join fails when - tbl join tbl_postfix (Yongzhi Chen, reviewed by Chaoyu Tang)
Date Mon, 19 Dec 2016 16:41:22 GMT
Repository: hive
Updated Branches:
  refs/heads/master 29cce163d -> 24577b6a3


HIVE-15437: avro tables join fails when - tbl join tbl_postfix (Yongzhi Chen, reviewed by
Chaoyu Tang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/24577b6a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/24577b6a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/24577b6a

Branch: refs/heads/master
Commit: 24577b6a34da7b955a0b94e508a4b6fedad856ff
Parents: 29cce16
Author: Yongzhi Chen <ychena@apache.org>
Authored: Thu Dec 15 11:00:02 2016 -0500
Committer: Yongzhi Chen <ychena@apache.org>
Committed: Mon Dec 19 11:26:03 2016 -0500

----------------------------------------------------------------------
 data/files/table1.avsc                          | 25 ++++++
 data/files/table1_1.avsc                        | 19 +++++
 .../ql/io/avro/AvroGenericRecordReader.java     |  7 +-
 .../test/queries/clientpositive/avrotblsjoin.q  | 28 +++++++
 .../results/clientpositive/avrotblsjoin.q.out   | 82 ++++++++++++++++++++
 5 files changed, 158 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/24577b6a/data/files/table1.avsc
----------------------------------------------------------------------
diff --git a/data/files/table1.avsc b/data/files/table1.avsc
new file mode 100644
index 0000000..2c96ad2
--- /dev/null
+++ b/data/files/table1.avsc
@@ -0,0 +1,25 @@
+{
+  "type" : "record",
+  "name" : "table1",
+  "doc" : "Sqoop import of table1",
+  "fields" : [ {
+    "name" : "col1",
+    "type" : [ "null", "string" ],
+    "default" : null,
+    "columnName" : "col1",
+    "sqlType" : "12"
+  }, {
+    "name" : "col2",
+    "type" : [ "null", "long" ],
+    "default" : null,
+    "columnName" : "col2",
+    "sqlType" : "13"
+  }, {
+    "name" : "col3",
+    "type" : [ "null", "string" ],
+    "default" : null,
+    "columnName" : "col3",
+    "sqlType" : "12"
+  } ],
+  "tableName" : "table1"
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/24577b6a/data/files/table1_1.avsc
----------------------------------------------------------------------
diff --git a/data/files/table1_1.avsc b/data/files/table1_1.avsc
new file mode 100644
index 0000000..1a7e518
--- /dev/null
+++ b/data/files/table1_1.avsc
@@ -0,0 +1,19 @@
+{
+  "type" : "record",
+  "name" : "table1_1",
+  "doc" : "Sqoop import of table1_1",
+  "fields" : [ {
+    "name" : "col1",
+    "type" : [ "null", "long" ],
+    "default" : null,
+    "columnName" : "col1",
+    "sqlType" : "13"
+  }, {
+    "name" : "col2",
+    "type" : [ "null", "string" ],
+    "default" : null,
+    "columnName" : "col2",
+    "sqlType" : "12"
+  }],
+  "tableName" : "table1_1"
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/24577b6a/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
index 4fccfc1..68138c8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/avro/AvroGenericRecordReader.java
@@ -32,6 +32,7 @@ import org.apache.avro.mapred.FsInput;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.PartitionDesc;
@@ -146,10 +147,10 @@ public class AvroGenericRecordReader implements
   private boolean pathIsInPartition(Path split, Path partitionPath) {
     boolean schemeless = split.toUri().getScheme() == null;
     if (schemeless) {
-      String schemelessPartitionPath = partitionPath.toUri().getPath();
-      return split.toString().startsWith(schemelessPartitionPath);
+      Path pathNoSchema = Path.getPathWithoutSchemeAndAuthority(partitionPath);
+      return FileUtils.isPathWithinSubtree(split,pathNoSchema);
     } else {
-      return split.toString().startsWith(partitionPath.toString());
+      return FileUtils.isPathWithinSubtree(split,partitionPath);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/24577b6a/ql/src/test/queries/clientpositive/avrotblsjoin.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/avrotblsjoin.q b/ql/src/test/queries/clientpositive/avrotblsjoin.q
new file mode 100644
index 0000000..8c1f084
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/avrotblsjoin.q
@@ -0,0 +1,28 @@
+drop table if exists table1;
+drop table if exists table1_1;
+
+dfs -cp ${system:hive.root}data/files/table1.avsc ${system:test.tmp.dir}/;
+dfs -cp ${system:hive.root}data/files/table1_1.avsc ${system:test.tmp.dir}/;
+
+create table table1
+   ROW FORMAT SERDE
+     'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+   STORED AS INPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+   OUTPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+   TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/table1.avsc');
+create table table1_1
+   ROW FORMAT SERDE
+     'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+   STORED AS INPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+   OUTPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+   TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/table1_1.avsc');
+insert into table1 values ("1", "2", "3");
+insert into table1_1 values (1, "2");
+set hive.auto.convert.join=false;
+set hive.strict.checks.type.safety=false;
+set hive.mapred.mode=nonstrict;
+select table1.col1, table1_1.* from table1 join table1_1 on table1.col1=table1_1.col1 where
table1_1.col1="1";

http://git-wip-us.apache.org/repos/asf/hive/blob/24577b6a/ql/src/test/results/clientpositive/avrotblsjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/avrotblsjoin.q.out b/ql/src/test/results/clientpositive/avrotblsjoin.q.out
new file mode 100644
index 0000000..d0170a3
--- /dev/null
+++ b/ql/src/test/results/clientpositive/avrotblsjoin.q.out
@@ -0,0 +1,82 @@
+PREHOOK: query: drop table if exists table1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists table1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists table1_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists table1_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table table1
+   ROW FORMAT SERDE
+     'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+   STORED AS INPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+   OUTPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table1
+POSTHOOK: query: create table table1
+   ROW FORMAT SERDE
+     'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+   STORED AS INPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+   OUTPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table1
+PREHOOK: query: create table table1_1
+   ROW FORMAT SERDE
+     'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+   STORED AS INPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+   OUTPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table1_1
+POSTHOOK: query: create table table1_1
+   ROW FORMAT SERDE
+     'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+   STORED AS INPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+   OUTPUTFORMAT
+     'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table1_1
+PREHOOK: query: insert into table1 values ("1", "2", "3")
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1
+POSTHOOK: query: insert into table1 values ("1", "2", "3")
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1
+POSTHOOK: Lineage: table1.col1 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1,
type:string, comment:), ]
+POSTHOOK: Lineage: table1.col2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2,
type:string, comment:), ]
+POSTHOOK: Lineage: table1.col3 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3,
type:string, comment:), ]
+PREHOOK: query: insert into table1_1 values (1, "2")
+PREHOOK: type: QUERY
+PREHOOK: Output: default@table1_1
+POSTHOOK: query: insert into table1_1 values (1, "2")
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@table1_1
+POSTHOOK: Lineage: table1_1.col1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1,
type:string, comment:), ]
+POSTHOOK: Lineage: table1_1.col2 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2,
type:string, comment:), ]
+WARNING: Comparing a bigint and a string may result in a loss of precision.
+Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a
cross product
+PREHOOK: query: select table1.col1, table1_1.* from table1 join table1_1 on table1.col1=table1_1.col1
where table1_1.col1="1"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select table1.col1, table1_1.* from table1 join table1_1 on table1.col1=table1_1.col1
where table1_1.col1="1"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table1_1
+#### A masked pattern was here ####
+1	1	2


Mime
View raw message