carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [1/3] incubator-carbondata git commit: extract command to common
Date Sat, 24 Dec 2016 14:13:03 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 956eb3348 -> 4a8e15d34


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b50866b3/integration/spark2/src/test/resources/data_alltypes.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/data_alltypes.csv b/integration/spark2/src/test/resources/data_alltypes.csv
new file mode 100644
index 0000000..2722edd
--- /dev/null
+++ b/integration/spark2/src/test/resources/data_alltypes.csv
@@ -0,0 +1,10 @@
+1,10,100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23 11:01:01,aaa
+5,17,140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27 11:01:02,bbb
+1,11,100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23 11:01:03,ccc
+1,10,150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24 11:01:04,ddd
+1,10,100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23 11:01:05,eeee
+3,14,160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26 11:01:06,ff
+2,10,100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23 11:01:07,ggg
+1,10,100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23 11:01:08,hhh
+4,16,130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23 11:01:09,iii
+1,10,100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23 11:01:10,jjj

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b50866b3/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala
b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala
index c8d5221..b7617e8 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala
@@ -21,10 +21,10 @@ package org.apache.carbondata.spark.testsuite.allqueries
 
 import java.io.File
 
-import org.apache.spark.sql.{Row, SaveMode}
-import org.apache.spark.sql.common.util.CarbonSessionTest._
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.common.util.QueryTest
 import org.scalatest.BeforeAndAfterAll
+
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.util.CarbonProperties
 
@@ -35,7 +35,7 @@ import org.apache.carbondata.core.util.CarbonProperties
 class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll {
 
   override def beforeAll {
-    clean
+    dropAllTable
     val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../../spark")
       .getCanonicalPath
 
@@ -57,14 +57,14 @@ class AllDataTypesTestCaseAggregate extends QueryTest with BeforeAndAfterAll
{
     sql("INSERT INTO table Carbon_automation_test select * from Carbon_automation_test_hive");
   }
 
-  def clean{
+  def dropAllTable{
     sql("drop table if exists Carbon_automation_test")
     sql("drop table if exists Carbon_automation_hive")
     sql("drop table if exists Carbon_automation_test_hive")
   }
 
   override def afterAll {
-    clean
+    dropAllTable
     CarbonProperties.getInstance()
       .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b50866b3/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
index bab78b9..057d894 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
@@ -17,8 +17,7 @@
 
 package org.apache.spark.carbondata
 
-import org.apache.spark.carbondata.util.QueryTest
-import org.apache.spark.carbondata.util.QueryTest._
+import org.apache.spark.sql.common.util.QueryTest
 import org.scalatest.BeforeAndAfterAll
 
 class CarbonDataSourceSuite extends QueryTest with BeforeAndAfterAll {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b50866b3/integration/spark2/src/test/scala/org/apache/spark/carbondata/util/QueryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/util/QueryTest.scala
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/util/QueryTest.scala
deleted file mode 100644
index 5a2a27e..0000000
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/util/QueryTest.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.spark.carbondata.util
-
-import java.io.File
-
-import org.apache.commons.io.FileUtils
-import org.apache.spark.sql.{CarbonEnv, SparkSession}
-import org.scalatest.FunSuite
-
-import org.apache.carbondata.core.util.CarbonProperties
-
-
-object QueryTest {
-
-  val (spark: SparkSession, storeLocation: String, warehouse: String, metastoredb: String)
=  {
-
-    val rootPath = new File(this.getClass.getResource("/").getPath
-                            + "../../../..").getCanonicalPath
-    val storeLocation = s"$rootPath/integration/spark2/target/store"
-    val warehouse = s"$rootPath/integration/spark2/target/warehouse"
-    val metastoredb = s"$rootPath/integration/spark2/target/metastore_db"
-
-    val spark = SparkSession
-      .builder()
-      .master("local")
-      .appName("Spark2Testcases")
-      .enableHiveSupport()
-      .config("spark.sql.warehouse.dir", warehouse)
-      .config("javax.jdo.option.ConnectionURL",
-        s"jdbc:derby:;databaseName=$metastoredb;create=true")
-      .getOrCreate()
-    spark.sparkContext.setLogLevel("ERROR")
-    CarbonProperties.getInstance()
-      .addProperty("carbon.kettle.home", s"$rootPath/processing/carbonplugins")
-      .addProperty("carbon.storelocation", storeLocation)
-
-    CarbonEnv.init(spark.sqlContext)
-    CarbonEnv.get.carbonMetastore.cleanStore()
-    (spark, storeLocation, warehouse, metastoredb)
-  }
-
-  def clean: Unit = {
-    val clean = (path: String) => FileUtils.deleteDirectory(new File(path))
-    clean(storeLocation)
-  }
-
-}
-
-class QueryTest extends FunSuite

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b50866b3/integration/spark2/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala
index ec241ae..6cce86a 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala
@@ -19,11 +19,7 @@
 
 package org.apache.spark.carbondata.vectorreader
 
-import java.io.File
-
-import org.apache.spark.carbondata.util.QueryTest
-import org.apache.spark.carbondata.util.QueryTest._
-import org.apache.spark.sql.execution.command.LoadTable
+import org.apache.spark.sql.common.util.QueryTest
 import org.apache.spark.sql.execution.{BatchedDataSourceScanExec, RowDataSourceScanExec}
 import org.scalatest.BeforeAndAfterAll
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b50866b3/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/CarbonSessionTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/CarbonSessionTest.scala
b/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/CarbonSessionTest.scala
deleted file mode 100644
index d29196e..0000000
--- a/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/CarbonSessionTest.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.common.util
-
-import java.io.File
-
-import org.apache.spark.sql.{DataFrame, SparkSession}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.commons.io.FileUtils
-
-object CarbonSessionTest extends{
-
-    val rootPath = new File(this.getClass.getResource("/").getPath
-      + "../../../..").getCanonicalPath
-    val storeLocation = s"$rootPath/examples/spark2/target/store"
-    val warehouse = s"$rootPath/examples/spark2/target/warehouse"
-    val metastoredb = s"$rootPath/examples/spark2/target/metastore_db"
-
-    val spark = {
-
-        // clean data folder
-        if (true) {
-            val clean = (path: String) => FileUtils.deleteDirectory(new File(path))
-            clean(storeLocation)
-            clean(warehouse)
-            clean(metastoredb)
-        }
-
-        val spark = SparkSession
-          .builder()
-          .master("local")
-          .appName("CarbonExample")
-          .enableHiveSupport()
-          .config("spark.sql.warehouse.dir", warehouse)
-          .config("javax.jdo.option.ConnectionURL",
-              s"jdbc:derby:;databaseName=$metastoredb;create=true")
-          .getOrCreate()
-
-        CarbonProperties.getInstance()
-          .addProperty("carbon.kettle.home", s"$rootPath/processing/carbonplugins")
-          .addProperty("carbon.storelocation", storeLocation)
-
-        spark.sparkContext.setLogLevel("WARN")
-
-        spark
-    }
-
-    val sc = spark.sparkContext
-
-    lazy val implicits = spark.implicits
-
-    def sql(sqlText: String): DataFrame  = spark.sql(sqlText)
-
-}
-
-

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b50866b3/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
b/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
index 44d3bfa..45dcb03 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala
@@ -17,14 +17,18 @@
 
 package org.apache.spark.sql.common.util
 
+import java.io.File
 import java.util.{Locale, TimeZone}
 
-import org.apache.carbondata.common.logging.LogServiceFactory
-
 import scala.collection.JavaConversions._
+
+import org.apache.commons.io.FileUtils
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.util._
-import org.apache.spark.sql.{DataFrame, Row, SQLContext}
+import org.apache.spark.sql.{DataFrame, Row, SQLContext, SparkSession}
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.util.CarbonProperties
 
 class QueryTest extends PlanTest {
 
@@ -35,6 +39,50 @@ class QueryTest extends PlanTest {
   // Add Locale setting
   Locale.setDefault(Locale.US)
 
+
+  val rootPath = new File(this.getClass.getResource("/").getPath + "../../../..").getCanonicalPath
+  val storeLocation = s"$rootPath/examples/spark2/target/store"
+  val warehouse = s"$rootPath/examples/spark2/target/warehouse"
+  val metastoredb = s"$rootPath/examples/spark2/target/metastore_db"
+
+  val spark = {
+    // clean data folder
+    if (true) {
+      val clean = (path: String) => FileUtils.deleteDirectory(new File(path))
+      clean(storeLocation)
+      clean(warehouse)
+      clean(metastoredb)
+    }
+
+    val spark = SparkSession
+        .builder()
+        .master("local")
+        .appName("CarbonExample")
+        .enableHiveSupport()
+        .config("spark.sql.warehouse.dir", warehouse)
+        .config("javax.jdo.option.ConnectionURL",
+          s"jdbc:derby:;databaseName=$metastoredb;create=true")
+        .getOrCreate()
+
+    CarbonProperties.getInstance()
+        .addProperty("carbon.kettle.home", s"$rootPath/processing/carbonplugins")
+        .addProperty("carbon.storelocation", storeLocation)
+
+    spark.sparkContext.setLogLevel("WARN")
+    spark
+  }
+
+  val sc = spark.sparkContext
+
+  lazy val implicits = spark.implicits
+
+  def sql(sqlText: String): DataFrame  = spark.sql(sqlText)
+
+  def clean: Unit = {
+    val clean = (path: String) => FileUtils.deleteDirectory(new File(path))
+    clean(storeLocation)
+  }
+
   /**
    * Runs the plan and makes sure the answer contains all of the keywords, or the
    * none of keywords are listed in the answer
@@ -79,6 +127,62 @@ class QueryTest extends PlanTest {
   protected def checkAnswer(df: DataFrame, expectedAnswer: DataFrame): Unit = {
     checkAnswer(df, expectedAnswer.collect())
   }
+
+
+  protected def createAndLoadInputTable(inputTableName: String, inputPath: String): Unit
= {
+    sql(
+      s"""
+         | CREATE TABLE $inputTableName
+         | (  shortField short,
+         |    intField int,
+         |    bigintField long,
+         |    doubleField double,
+         |    stringField string,
+         |    timestampField string,
+         |    decimalField decimal(18,2),
+         |    dateField string,
+         |    charField char(5)
+         | )
+         | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+       """.stripMargin)
+
+    sql(
+      s"""
+         | LOAD DATA LOCAL INPATH '$inputPath'
+         | INTO TABLE $inputTableName
+       """.stripMargin)
+  }
+
+  protected def createAndLoadTestTable(tableName: String, inputTableName: String): Unit =
{
+    sql(
+      s"""
+         | CREATE TABLE $tableName(
+         |    shortField short,
+         |    intField int,
+         |    bigintField long,
+         |    doubleField double,
+         |    stringField string,
+         |    timestampField timestamp,
+         |    decimalField decimal(18,2),
+         |    dateField date,
+         |    charField char(5)
+         | )
+         | USING org.apache.spark.sql.CarbonSource
+         | OPTIONS ('tableName' '$tableName')
+       """.stripMargin)
+    sql(
+      s"""
+         | INSERT INTO TABLE $tableName
+         | SELECT shortField, intField, bigintField, doubleField, stringField,
+         | from_unixtime(unix_timestamp(timestampField,'yyyy/M/dd')) timestampField, decimalField,
+         | cast(to_date(from_unixtime(unix_timestamp(dateField,'yyyy/M/dd'))) as date), charField
+         | FROM $inputTableName
+       """.stripMargin)
+  }
+
+  protected def dropTable(tableName: String): Unit ={
+    sql(s"DROP TABLE IF EXISTS $tableName")
+  }
 }
 
 object QueryTest {
@@ -146,4 +250,5 @@ object QueryTest {
 
     return None
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b50866b3/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
b/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
new file mode 100644
index 0000000..f80b12d
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import java.io.File
+import java.sql.Timestamp
+import java.util.Date
+
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.api.CarbonStore
+import org.apache.carbondata.core.util.CarbonUtil
+
+class CarbonCommandSuite extends QueryTest with BeforeAndAfterAll {
+
+  override def beforeAll(): Unit = {
+    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../../spark2")
+        .getCanonicalPath
+    dropTable("csv_table")
+    createAndLoadInputTable("csv_table", s"$currentDirectory/src/test/resources/data_alltypes.csv")
+    createAndLoadTestTable("carbon_table", "csv_table")
+  }
+
+  override def afterAll(): Unit = {
+    dropTable("csv_table")
+    dropTable("carbon_table")
+  }
+
+  test("show segment") {
+    ShowSegments.main(Array(s"${CarbonUtil.getCarbonStorePath}", "carbon_table"))
+  }
+
+  test("delete segment by id") {
+    DeleteSegmentById.main(Array(s"${CarbonUtil.getCarbonStorePath}", "carbon_table", "0"))
+    assert(!CarbonStore.isSegmentValid("default", "carbon_table", "0"))
+  }
+
+  test("delete segment by date") {
+    createAndLoadTestTable("carbon_table2", "csv_table")
+    val time = new Timestamp(new Date().getTime)
+    DeleteSegmentByDate.main(Array(s"${CarbonUtil.getCarbonStorePath}", "carbon_table2",
time.toString))
+    assert(!CarbonStore.isSegmentValid("default", "carbon_table2", "0"))
+    dropTable("carbon_table2")
+  }
+
+  test("clean files") {
+    val table = "carbon_table3"
+    createAndLoadTestTable(table, "csv_table")
+    ShowSegments.main(Array(s"${CarbonUtil.getCarbonStorePath}", table))
+    DeleteSegmentById.main(Array(s"${CarbonUtil.getCarbonStorePath}", table, "0"))
+    ShowSegments.main(Array(s"${CarbonUtil.getCarbonStorePath}", table))
+    CleanFiles.main(Array(s"${CarbonUtil.getCarbonStorePath}", table))
+    ShowSegments.main(Array(s"${CarbonUtil.getCarbonStorePath}", table))
+    val tablePath = s"${CarbonUtil.getCarbonStorePath}${File.separator}default${File.separator}$table"
+    val f = new File(s"$tablePath/Fact/Part0")
+    assert(f.isDirectory)
+
+    // all segment folders should be deleted after CleanFiles command
+    assert(f.list().length == 0)
+    dropTable(table)
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b50866b3/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java
b/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java
index a3bc3a3..030ee0b 100644
--- a/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java
+++ b/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java
@@ -589,5 +589,9 @@ public class SegmentStatusManager {
     public List<String> getInvalidSegments() {
       return invalidSegments;
     }
+
+    public boolean isValid(String segment) {
+      return validSegments.contains(segment);
+    }
   }
 }


Mime
View raw message