carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [1/2] incubator-carbondata git commit: update the examples
Date Fri, 28 Apr 2017 09:14:38 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master b728b8f94 -> 472f9c9d5


update the examples

just update examples


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2e8ef57d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2e8ef57d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2e8ef57d

Branch: refs/heads/master
Commit: 2e8ef57d5a72e7c35fbf851ca28f114501028d17
Parents: b728b8f
Author: chenerlu <chenerlu@huawei.com>
Authored: Thu Apr 27 19:20:09 2017 +0800
Committer: chenliang613 <chenliang613@huawei.com>
Committed: Fri Apr 28 17:12:57 2017 +0800

----------------------------------------------------------------------
 .../carbondata/examples/AlterTableExample.scala |  32 ++--
 .../examples/CarbonDataFrameExample.scala       |   4 +-
 .../examples/CarbonSessionExample.scala         | 114 +++++++-------
 .../examples/SparkSessionExample.scala          | 156 +++++++++----------
 4 files changed, 158 insertions(+), 148 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
index f37a83d..dd2a28a 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
@@ -24,6 +24,10 @@ import org.apache.spark.sql.SparkSession
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.util.CarbonProperties
 
+/**
+ * For alter table relative syntax, you can refer to DDL operation
+ * document (ddl-operation-on-carbondata.md)
+ */
 object AlterTableExample {
 
   def main(args: Array[String]): Unit = {
@@ -55,17 +59,17 @@ object AlterTableExample {
     spark.sql(
       s"""
          | CREATE TABLE carbon_table(
-         |    shortField short,
-         |    intField int,
-         |    bigintField long,
-         |    doubleField double,
-         |    stringField string,
-         |    timestampField timestamp,
-         |    decimalField decimal(18,2),
-         |    dateField date,
-         |    charField char(5),
-         |    floatField float,
-         |    complexData array<string>
+         | shortField SHORT,
+         | intField INT,
+         | bigintField LONG,
+         | doubleField DOUBLE,
+         | stringField STRING,
+         | timestampField TIMESTAMP,
+         | decimalField DECIMAL(18,2),
+         | dateField DATE,
+         | charField CHAR(5),
+         | floatField FLOAT,
+         | complexData ARRAY<STRING>
          | )
          | STORED BY 'carbondata'
          | TBLPROPERTIES('DICTIONARY_INCLUDE'='dateField, charField')
@@ -73,11 +77,11 @@ object AlterTableExample {
 
     // Alter table change data type
     spark.sql("DESCRIBE FORMATTED carbon_table").show()
-    spark.sql("ALTER TABLE carbon_table CHANGE intField intField bigint").show()
+    spark.sql("ALTER TABLE carbon_table CHANGE intField intField BIGINT").show()
 
     // Alter table add columns
     spark.sql("DESCRIBE FORMATTED carbon_table").show()
-    spark.sql("ALTER TABLE carbon_table ADD COLUMNS (newField String) " +
+    spark.sql("ALTER TABLE carbon_table ADD COLUMNS (newField STRING) " +
               "TBLPROPERTIES ('DEFAULT.VALUE.newField'='def')").show()
 
     // Alter table drop columns
@@ -93,5 +97,7 @@ object AlterTableExample {
     spark.sql("DROP TABLE IF EXISTS carbon_table")
     spark.sql("DROP TABLE IF EXISTS new_carbon_table")
 
+    spark.stop()
+
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
index e4d1646..74740e7 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
@@ -24,7 +24,6 @@ import org.apache.spark.sql.{SaveMode, SparkSession}
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.util.CarbonProperties
 
-// scalastyle:off println
 object CarbonDataFrameExample {
 
   def main(args: Array[String]) {
@@ -84,6 +83,7 @@ object CarbonDataFrameExample {
     carbondf.filter($"number" > 31).show()
 
     spark.sql("DROP TABLE IF EXISTS carbon_table")
+
+    spark.stop()
   }
 }
-// scalastyle:on println

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index 2ae9341..371b9bb 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -53,17 +53,17 @@ object CarbonSessionExample {
     spark.sql(
       s"""
          | CREATE TABLE carbon_table(
-         |    shortField short,
-         |    intField int,
-         |    bigintField long,
-         |    doubleField double,
-         |    stringField string,
-         |    timestampField timestamp,
-         |    decimalField decimal(18,2),
-         |    dateField date,
-         |    charField char(5),
-         |    floatField float,
-         |    complexData array<string>
+         | shortField SHORT,
+         | intField INT,
+         | bigintField LONG,
+         | doubleField DOUBLE,
+         | stringField STRING,
+         | timestampField TIMESTAMP,
+         | decimalField DECIMAL(18,2),
+         | dateField DATE,
+         | charField CHAR(5),
+         | floatField FLOAT,
+         | complexData ARRAY<STRING>
          | )
          | STORED BY 'carbondata'
          | TBLPROPERTIES('DICTIONARY_INCLUDE'='dateField, charField')
@@ -76,63 +76,69 @@ object CarbonSessionExample {
       s"""
          | LOAD DATA LOCAL INPATH '$path'
          | INTO TABLE carbon_table
-         | options('FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData','COMPLEX_DELIMITER_LEVEL_1'='#')
+         | OPTIONS('FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData',
+         | 'COMPLEX_DELIMITER_LEVEL_1'='#')
        """.stripMargin)
     // scalastyle:on
 
-    spark.sql("""
-             SELECT *
-             FROM carbon_table
-             where stringfield = 'spark' and decimalField > 40
-              """).show
+    spark.sql(
+      s"""
+        | SELECT *
+        | FROM carbon_table
+        | WHERE stringfield = 'spark' AND decimalField > 40
+      """.stripMargin).show()
 
-    spark.sql("""
-             SELECT *
-             FROM carbon_table where length(stringField) = 5
-              """).show
+    spark.sql(
+      s"""
+         | SELECT *
+         | FROM carbon_table WHERE length(stringField) = 5
+       """.stripMargin).show()
+
+    spark.sql(
+      s"""
+         | SELECT *
+         | FROM carbon_table WHERE date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
+       """.stripMargin).show()
 
-    spark.sql("""
-             SELECT *
-             FROM carbon_table where date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
-              """).show
+    spark.sql("SELECT count(stringField) FROM carbon_table").show()
 
-    spark.sql("""
-             select count(stringField) from carbon_table
-              """.stripMargin).show
+    spark.sql(
+      s"""
+         | SELECT sum(intField), stringField
+         | FROM carbon_table
+         | GROUP BY stringField
+       """.stripMargin).show()
 
-    spark.sql("""
-           SELECT sum(intField), stringField
-           FROM carbon_table
-           GROUP BY stringField
-              """).show
+    spark.sql(
+      s"""
+        | SELECT t1.*, t2.*
+        | FROM carbon_table t1, carbon_table t2
+        | WHERE t1.stringField = t2.stringField
+      """.stripMargin).show()
 
     spark.sql(
-      """
-        |select t1.*, t2.*
-        |from carbon_table t1, carbon_table t2
-        |where t1.stringField = t2.stringField
-      """.stripMargin).show
+      s"""
+        | WITH t1 AS (
+        | SELECT * FROM carbon_table
+        | UNION ALL
+        | SELECT * FROM carbon_table
+        | )
+        | SELECT t1.*, t2.*
+        | FROM t1, carbon_table t2
+        | WHERE t1.stringField = t2.stringField
+      """.stripMargin).show()
 
     spark.sql(
-      """
-        |with t1 as (
-        |select * from carbon_table
-        |union all
-        |select * from carbon_table
-        |)
-        |select t1.*, t2.*
-        |from t1, carbon_table t2
-        |where t1.stringField = t2.stringField
-      """.stripMargin).show
-
-    spark.sql("""
-             SELECT *
-             FROM carbon_table
-             where stringfield = 'spark' and floatField > 2.8
-              """).show
+      s"""
+         | SELECT *
+         | FROM carbon_table
+         | WHERE stringField = 'spark' and floatField > 2.8
+       """.stripMargin).show()
 
     // Drop table
     spark.sql("DROP TABLE IF EXISTS carbon_table")
+
+    spark.stop()
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
index 63e680a..b823be5 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
@@ -30,7 +30,7 @@ object SparkSessionExample {
 
   def main(args: Array[String]): Unit = {
     val rootPath = new File(this.getClass.getResource("/").getPath
-        + "../../../..").getCanonicalPath
+                            + "../../../..").getCanonicalPath
     val storeLocation = s"$rootPath/examples/spark2/target/store"
     val warehouse = s"$rootPath/examples/spark2/target/warehouse"
     val metastoredb = s"$rootPath/examples/spark2/target/metastore_db"
@@ -44,14 +44,14 @@ object SparkSessionExample {
     }
 
     val spark = SparkSession
-        .builder()
-        .master("local")
-        .appName("SparkSessionExample")
-        .enableHiveSupport()
-        .config("spark.sql.warehouse.dir", warehouse)
-        .config("javax.jdo.option.ConnectionURL",
-          s"jdbc:derby:;databaseName=$metastoredb;create=true")
-        .getOrCreate()
+      .builder()
+      .master("local")
+      .appName("SparkSessionExample")
+      .enableHiveSupport()
+      .config("spark.sql.warehouse.dir", warehouse)
+      .config("javax.jdo.option.ConnectionURL",
+        s"jdbc:derby:;databaseName=$metastoredb;create=true")
+      .getOrCreate()
 
     CarbonProperties.getInstance()
       .addProperty("carbon.storelocation", storeLocation)
@@ -59,46 +59,43 @@ object SparkSessionExample {
     spark.sparkContext.setLogLevel("WARN")
 
     CarbonProperties.getInstance()
-        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
-        .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+      .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
 
     // Create table
     spark.sql(
       s"""
          | CREATE TABLE carbon_table(
-         |    shortField short,
-         |    intField int,
-         |    bigintField long,
-         |    doubleField double,
-         |    stringField string,
-         |    timestampField timestamp,
-         |    decimalField decimal(18,2),
-         |    dateField date,
-         |    charField char(5)
+         | shortField SHORT,
+         | intField INT,
+         | bigintField LONG,
+         | doubleField DOUBLE,
+         | stringField STRING,
+         | timestampField TIMESTAMP,
+         | decimalField DECIMAL(18,2),
+         | dateField DATE,
+         | charField CHAR(5)
          | )
          | USING org.apache.spark.sql.CarbonSource
          | OPTIONS('DICTIONARY_INCLUDE'='dateField, charField',
-         |   'dbName'='default', 'tableName'='carbon_table')
+         | 'dbName'='default', 'tableName'='carbon_table')
        """.stripMargin)
 
-    // val prop = s"$rootPath/conf/dataload.properties.template"
-    // val tableName = "carbon_table"
     val path = s"$rootPath/examples/spark2/src/main/resources/data.csv"
-    // TableLoader.main(Array[String](prop, tableName, path))
 
     spark.sql(
       s"""
-         | CREATE TABLE csv_table
-         | (  shortField short,
-         |    intField int,
-         |    bigintField long,
-         |    doubleField double,
-         |    stringField string,
-         |    timestampField string,
-         |    decimalField decimal(18,2),
-         |    dateField string,
-         |    charField char(5))
-         |    ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+         | CREATE TABLE csv_table(
+         | shortField SHORT,
+         | intField INT,
+         | bigintField LONG,
+         | doubleField DOUBLE,
+         | stringField STRING,
+         | timestampField STRING,
+         | decimalField DECIMAL(18,2),
+         | dateField STRING,
+         | charField CHAR(5))
+         | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
        """.stripMargin)
 
     spark.sql(
@@ -107,10 +104,7 @@ object SparkSessionExample {
          | INTO TABLE csv_table
        """.stripMargin)
 
-    spark.sql("""
-             SELECT *
-             FROM csv_table
-              """).show
+    spark.sql("SELECT * FROM csv_table").show()
 
     spark.sql(
       s"""
@@ -121,56 +115,60 @@ object SparkSessionExample {
          | FROM csv_table
        """.stripMargin)
 
-    spark.sql("""
-             SELECT *
-             FROM carbon_table
-             where stringfield = 'spark' and decimalField > 40
-              """).show
+    spark.sql(
+      s"""
+         | SELECT *
+         | FROM carbon_table
+         | WHERE stringfield = 'spark' AND decimalField > 40
+      """.stripMargin).show()
 
     // Shows with raw data's timestamp format
-    spark.sql("""
-             SELECT
-             stringField, date_format(timestampField, "yyyy/MM/dd HH:mm:ss") as timestampField
-             FROM carbon_table where length(stringField) = 5
-              """).show
-
-    spark.sql("""
-             SELECT *
-             FROM carbon_table where date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
-              """).show
-
-    spark.sql("""
-             select count(stringField) from carbon_table
-              """.stripMargin).show
-
-    spark.sql("""
-           SELECT sum(intField), stringField
-           FROM carbon_table
-           GROUP BY stringField
-           """).show
+    spark.sql(
+      s"""
+         | SELECT
+         | stringField, date_format(timestampField, "yyyy/MM/dd HH:mm:ss") AS
+         | timestampField
+         | FROM carbon_table WHERE length(stringField) = 5
+       """.stripMargin).show()
 
     spark.sql(
-      """
-        |select t1.*, t2.*
-        |from carbon_table t1, carbon_table t2
-        |where t1.stringField = t2.stringField
-      """.stripMargin).show
+      s"""
+         | SELECT *
+         | FROM carbon_table where date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
+       """.stripMargin).show()
+
+    spark.sql("SELECT count(stringField) FROM carbon_table").show()
 
     spark.sql(
-      """
-        |with t1 as (
-        |select * from carbon_table
-        |union all
-        |select * from carbon_table
-        |)
-        |select t1.*, t2.*
-        |from t1, carbon_table t2
-        |where t1.stringField = t2.stringField
-      """.stripMargin).show
+      s"""
+         | SELECT sum(intField), stringField
+         | FROM carbon_table
+         | GROUP BY stringField
+       """.stripMargin).show()
+
+    spark.sql(
+      s"""
+         | SELECT t1.*, t2.*
+         | FROM carbon_table t1, carbon_table t2
+         | WHERE t1.stringField = t2.stringField
+      """.stripMargin).show()
+
+    spark.sql(
+      s"""
+         | WITH t1 AS (
+         | SELECT * FROM carbon_table
+         | UNION ALL
+         | SELECT * FROM carbon_table
+         | )
+         | SELECT t1.*, t2.*
+         | FROM t1, carbon_table t2
+         | WHERE t1.stringField = t2.stringField
+      """.stripMargin).show()
 
     // Drop table
     spark.sql("DROP TABLE IF EXISTS carbon_table")
     spark.sql("DROP TABLE IF EXISTS csv_table")
+
     spark.stop()
   }
 }


Mime
View raw message