carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-2467] sdk writer log shouldnot print null
Date Fri, 18 May 2018 08:52:54 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 201804858 -> 78efc7f71


[CARBONDATA-2467] sdk writer log shouldnot print null

issue :

Logger prints the threadname along with log. In case of sdk writer we are assigning null for
dbname and table name and table name is used to assign the thread name in CarbonTableOutputFormat
and DataWriterProcessStepImpl. which leads to add null in many logs.
Comparision of two array type was being wrong.
Solution :

assign temp value with current time in millisec to tablename instead of null.
ArrayType class donot contains equls method. Hence it goes to Object.equals() which will perform
== operation to compare both the object which is wrong . so add equals method to ArrayType
class .

This closes #2292


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/78efc7f7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/78efc7f7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/78efc7f7

Branch: refs/heads/master
Commit: 78efc7f7110e152b8f3d2ec833e13278356a6a2d
Parents: 2018048
Author: rahulforallp <rahul.kumar@knoldus.in>
Authored: Thu May 10 14:37:44 2018 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Fri May 18 14:22:44 2018 +0530

----------------------------------------------------------------------
 .../core/metadata/datatype/ArrayType.java       | 13 +++
 .../TestNonTransactionalCarbonTable.scala       | 84 ++++++++++++++++++++
 .../sdk/file/CarbonWriterBuilder.java           |  4 +-
 3 files changed, 99 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/78efc7f7/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
index c327d7f..0f0f120 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
@@ -34,4 +34,17 @@ public class ArrayType extends DataType {
   public DataType getElementType() {
     return elementType;
   }
+
+  @Override public boolean equals(Object o) {
+    if (this == o) return true;
+    if (!(o instanceof ArrayType)) return false;
+
+    ArrayType arrayType = (ArrayType) o;
+
+    return elementType.equals(arrayType.elementType);
+  }
+
+  @Override public int hashCode() {
+    return elementType.hashCode();
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/78efc7f7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index cc3cbb5..a15d0c2 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -1331,6 +1331,90 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
     buildAvroTestDataStructOfArray(3, null)
   }
 
+  // ArrayOfStruct test
+  def buildAvroTestDataArrayOfStructWithNoSortCol(rows: Int, options: util.Map[String, String]):
Any = {
+    FileUtils.deleteDirectory(new File(writerPath))
+
+    val mySchema = """ {
+                     |	"name": "address",
+                     |	"type": "record",
+                     |	"fields": [
+                     |		{
+                     |			"name": "exp",
+                     |			"type": "int"
+                     |		},
+                     |		{
+                     |			"name": "age",
+                     |			"type": "int"
+                     |		},
+                     |		{
+                     |			"name": "doorNum",
+                     |			"type": {
+                     |				"type": "array",
+                     |				"items": {
+                     |					"type": "record",
+                     |					"name": "my_address",
+                     |					"fields": [
+                     |						{
+                     |							"name": "street",
+                     |							"type": "string"
+                     |						},
+                     |						{
+                     |							"name": "city",
+                     |							"type": "string"
+                     |						}
+                     |					]
+                     |				}
+                     |			}
+                     |		}
+                     |	]
+                     |} """.stripMargin
+    val json =
+      """ {"exp":5,"age":10,"doorNum" :
+        |[{"street":"abc","city":"city1"},
+        |{"street":"def","city":"city2"},
+        |{"street":"ghi","city":"city3"},
+        |{"street":"jkl","city":"city4"}]} """.stripMargin
+
+    val fields = new Array[Field](3)
+    fields(0) = new Field("exp", DataTypes.INT)
+    fields(1) = new Field("age", DataTypes.INT)
+
+    val fld = new util.ArrayList[StructField]
+    fld.add(new StructField("street", DataTypes.STRING))
+    fld.add(new StructField("city", DataTypes.STRING))
+
+    val fld2 = new util.ArrayList[StructField]
+    fld2.add(new StructField("my_address", DataTypes.createStructType(fld), fld))
+    fields(2) = new Field("doorNum", DataTypes.createArrayType(fld2.get(0).getDataType),
fld2)
+
+    WriteFilesWithAvroWriter(rows, mySchema, json, fields)
+  }
+
+  test("Read sdk writer Avro output Record Type with no sort columns") {
+    buildAvroTestDataArrayOfStructWithNoSortCol(3,null)
+    assert(new File(writerPath).exists())
+    sql("DROP TABLE IF EXISTS sdkOutputTable")
+    sql(
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY 'carbondata' LOCATION
+         |'$writerPath' """.stripMargin)
+
+    sql("desc formatted sdkOutputTable").show(false)
+    sql("select * from sdkOutputTable").show(false)
+
+    /*
+    +---+---+----------------------------------------------------+
+    |exp|age|doorNum                                             |
+    +---+---+----------------------------------------------------+
+    |5  |10 |[[abc,city1], [def,city2], [ghi,city3], [jkl,city4]]|
+    |5  |10 |[[abc,city1], [def,city2], [ghi,city3], [jkl,city4]]|
+    |5  |10 |[[abc,city1], [def,city2], [ghi,city3], [jkl,city4]]|
+    +---+---+----------------------------------------------------+
+    */
+    sql("DROP TABLE sdkOutputTable")
+    // drop table should not delete the files
+    assert(new File(writerPath).listFiles().length > 0)
+  }
 
   test("Read sdk writer Avro output Record Type") {
     buildAvroTestDataStructType()

http://git-wip-us.apache.org/repos/asf/carbondata/blob/78efc7f7/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index 36be65f..0f9c9d7 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -414,8 +414,8 @@ public class CarbonWriterBuilder {
       tableName = "_tempTable";
       dbName = "_tempDB";
     } else {
-      dbName = null;
-      tableName = null;
+      dbName = "";
+      tableName = "_tempTable_" + String.valueOf(UUID);
     }
     TableSchema schema = tableSchemaBuilder.build();
     schema.setTableName(tableName);


Mime
View raw message