carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: fix issue carbondata-339
Date Fri, 28 Oct 2016 00:40:36 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 5a97fc795 -> ab9c6c083


fix issue carbondata-339


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/64d4d6da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/64d4d6da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/64d4d6da

Branch: refs/heads/master
Commit: 64d4d6daaf6e8adede6cfffe94221d20f365631c
Parents: 5a97fc7
Author: hseagle <hsxupeng@gmail.com>
Authored: Thu Oct 27 10:55:53 2016 +0800
Committer: hseagle <hsxupeng@gmail.com>
Committed: Thu Oct 27 10:55:53 2016 +0800

----------------------------------------------------------------------
 .../carbondata/spark/util/GlobalDictionaryUtil.scala    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/64d4d6da/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 09c32c8..a1f4400 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -742,13 +742,13 @@ object GlobalDictionaryUtil extends Logging {
    */
   def generateGlobalDictionary(sqlContext: SQLContext,
                                carbonLoadModel: CarbonLoadModel,
-                               hdfsLocation: String,
+                               storePath: String,
                                dataFrame: Option[DataFrame] = None): Unit = {
     try {
       val table = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getAbsoluteTableIdentifier
         .getCarbonTableIdentifier
       // create dictionary folder if not exists
-      val carbonTablePath = CarbonStorePath.getCarbonTablePath(hdfsLocation, table)
+      val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, table)
       val dictfolderPath = carbonTablePath.getMetadataDirectoryPath
       // columns which need to generate global dictionary file
       val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
@@ -777,7 +777,7 @@ object GlobalDictionaryUtil extends Logging {
         if (colDictFilePath != null) {
           // generate predefined dictionary
           generatePredefinedColDictionary(colDictFilePath, table,
-            dimensions, carbonLoadModel, sqlContext, hdfsLocation, dictfolderPath)
+            dimensions, carbonLoadModel, sqlContext, storePath, dictfolderPath)
         }
         if (headers.length > df.columns.length) {
           val msg = "The number of columns in the file header do not match the number of
" +
@@ -792,7 +792,7 @@ object GlobalDictionaryUtil extends Logging {
           // select column to push down pruning
           df = df.select(requireColumnNames.head, requireColumnNames.tail: _*)
           val model = createDictionaryLoadModel(carbonLoadModel, table, requireDimension,
-            hdfsLocation, dictfolderPath, false)
+            storePath, dictfolderPath, false)
           // combine distinct value in a block and partition by column
           val inputRDD = new CarbonBlockDistinctValuesCombineRDD(df.rdd, model)
             .partitionBy(new ColumnPartitioner(model.primDimensions.length))
@@ -815,7 +815,7 @@ object GlobalDictionaryUtil extends Logging {
               dimDataframe = dimDataframe.select(requireColumnNamesForDim.head,
                 requireColumnNamesForDim.tail: _*)
               val modelforDim = createDictionaryLoadModel(carbonLoadModel, table,
-                requireDimensionForDim, hdfsLocation, dictfolderPath, false)
+                requireDimensionForDim, storePath, dictfolderPath, false)
               val inputRDDforDim = new CarbonBlockDistinctValuesCombineRDD(
                 dimDataframe.rdd, modelforDim)
                 .partitionBy(new ColumnPartitioner(modelforDim.primDimensions.length))
@@ -842,7 +842,7 @@ object GlobalDictionaryUtil extends Logging {
             pruneDimensions(dimensions, headers, headers)
           if (requireDimension.nonEmpty) {
             val model = createDictionaryLoadModel(carbonLoadModel, table, requireDimension,
-              hdfsLocation, dictfolderPath, false)
+              storePath, dictfolderPath, false)
             // check if dictionary files contains bad record
             val accumulator = sqlContext.sparkContext.accumulator(0)
             // read local dictionary file, and group by key


Mime
View raw message