carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: remove unused dimension table dict
Date Thu, 29 Dec 2016 07:41:39 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master fd9d1026c -> aaf98edca


remove unused dimension table dict


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ff1bde14
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ff1bde14
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ff1bde14

Branch: refs/heads/master
Commit: ff1bde146f40551409708285e2b69b75837f20ba
Parents: fd9d102
Author: Jay357089 <liujunjie9@huawei.com>
Authored: Mon Dec 26 20:43:23 2016 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Thu Dec 29 15:28:34 2016 +0800

----------------------------------------------------------------------
 .../carbondata/spark/load/CarbonLoaderUtil.java |  2 -
 .../spark/util/GlobalDictionaryUtil.scala       | 25 ---------
 .../execution/command/carbonTableSchema.scala   |  6 --
 .../spark/util/AllDictionaryTestCase.scala      |  6 +-
 .../AutoHighCardinalityIdentifyTestCase.scala   |  2 -
 ...GlobalDictionaryUtilConcurrentTestCase.scala |  4 +-
 .../util/GlobalDictionaryUtilTestCase.scala     |  8 +--
 .../execution/command/carbonTableSchema.scala   |  6 --
 .../processing/csvload/DataGraphExecuter.java   | 59 --------------------
 .../dataprocessor/DataProcessTaskStatus.java    | 13 -----
 .../dataprocessor/IDataProcessStatus.java       |  4 --
 .../processing/model/CarbonLoadModel.java       | 17 ------
 12 files changed, 5 insertions(+), 147 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 0d2ab6f..56ddce2 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -179,7 +179,6 @@ public final class CarbonLoaderUtil {
     DataProcessTaskStatus dataProcessTaskStatus
             = new DataProcessTaskStatus(databaseName, tableName);
     dataProcessTaskStatus.setCsvFilePath(loadModel.getFactFilePath());
-    dataProcessTaskStatus.setDimCSVDirLoc(loadModel.getDimFolderPath());
     if (loadModel.isDirectLoad()) {
       dataProcessTaskStatus.setFilesToProcess(loadModel.getFactFilesToProcess());
       dataProcessTaskStatus.setDirectLoad(true);
@@ -194,7 +193,6 @@ public final class CarbonLoaderUtil {
     dataProcessTaskStatus.setRddIteratorKey(loadModel.getRddIteratorKey());
     dataProcessTaskStatus.setDateFormat(loadModel.getDateFormat());
     SchemaInfo info = new SchemaInfo();
-
     info.setDatabaseName(databaseName);
     info.setTableName(tableName);
     info.setAutoAggregateRequest(loadModel.isAggLoadRequest());

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index e578488..3274e58 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -772,31 +772,6 @@ object GlobalDictionaryUtil {
         } else {
           LOGGER.info("No column found for generating global dictionary in source data files")
         }
-        // generate global dict from dimension file
-        if (carbonLoadModel.getDimFolderPath != null) {
-          val fileMapArray = carbonLoadModel.getDimFolderPath.split(",")
-          for (fileMap <- fileMapArray) {
-            val dimTableName = fileMap.split(":")(0)
-            var dimDataframe = loadDataFrame(sqlContext, carbonLoadModel)
-            val (requireDimensionForDim, requireColumnNamesForDim) =
-              pruneDimensions(dimensions, dimDataframe.columns, dimDataframe.columns)
-            if (requireDimensionForDim.length >= 1) {
-              dimDataframe = dimDataframe.select(requireColumnNamesForDim.head,
-                requireColumnNamesForDim.tail: _*)
-              val modelforDim = createDictionaryLoadModel(carbonLoadModel, carbonTableIdentifier,
-                requireDimensionForDim, storePath, dictfolderPath, false)
-              val inputRDDforDim = new CarbonBlockDistinctValuesCombineRDD(
-                dimDataframe.rdd, modelforDim)
-                .partitionBy(new ColumnPartitioner(modelforDim.primDimensions.length))
-              val statusListforDim = new CarbonGlobalDictionaryGenerateRDD(
-                inputRDDforDim, modelforDim).collect()
-              checkStatus(carbonLoadModel, sqlContext, modelforDim, statusListforDim)
-            } else {
-              LOGGER.info(s"No columns in dimension table $dimTableName " +
-                          "to generate global dictionary")
-            }
-          }
-        }
       } else {
         LOGGER.info("Generate global dictionary from dictionary files!")
         val isNonempty = validateAllDictionaryPath(allDictionaryPath)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 8ad1203..d13346a 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -304,12 +304,6 @@ case class LoadTable(
       carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
       carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
       carbonLoadModel.setStorePath(relation.tableMeta.storePath)
-      if (dimFilesPath.isEmpty) {
-        carbonLoadModel.setDimFolderPath(null)
-      } else {
-        val x = dimFilesPath.map(f => f.table + ":" + CarbonUtil.checkAndAppendHDFSUrl(f.loadPath))
-        carbonLoadModel.setDimFolderPath(x.mkString(","))
-      }
 
       val table = relation.tableMeta.carbonTable
       carbonLoadModel.setAggTables(table.getAggregateTablesName.asScala.toArray)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index 90cf00a..24e478f 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -43,7 +43,6 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
 
   def buildCarbonLoadModel(relation: CarbonRelation,
     filePath: String,
-    dimensionFilePath: String,
     header: String,
     allDictFilePath: String): CarbonLoadModel = {
     val carbonLoadModel = new CarbonLoadModel
@@ -55,7 +54,6 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
     carbonLoadModel.setTableName(table.getFactTableName)
     carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
     carbonLoadModel.setFactFilePath(filePath)
-    carbonLoadModel.setDimFolderPath(dimensionFilePath)
     carbonLoadModel.setCsvHeader(header)
     carbonLoadModel.setCsvDelimiter(",")
     carbonLoadModel.setComplexDelimiterLevel1("\\$")
@@ -115,7 +113,7 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
 
   test("Support generate global dictionary from all dictionary files") {
     val header = "id,name,city,age"
-    val carbonLoadModel = buildCarbonLoadModel(sampleRelation, null, null, header, sampleAllDictionaryFile)
+    val carbonLoadModel = buildCarbonLoadModel(sampleRelation, null, header, sampleAllDictionaryFile)
     GlobalDictionaryUtil
       .generateGlobalDictionary(CarbonHiveContext,
         carbonLoadModel,
@@ -127,7 +125,7 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
 
   test("Support generate global dictionary from all dictionary files for complex type") {
     val header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber"
-    val carbonLoadModel = buildCarbonLoadModel(complexRelation, null, null, header, complexAllDictionaryFile)
+    val carbonLoadModel = buildCarbonLoadModel(complexRelation, null, header, complexAllDictionaryFile)
     GlobalDictionaryUtil
       .generateGlobalDictionary(CarbonHiveContext,
       carbonLoadModel,

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
index e92561e..5c61ab8 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
@@ -46,7 +46,6 @@ class AutoHighCardinalityIdentifyTestCase extends QueryTest with BeforeAndAfterA
 
   def buildCarbonLoadModel(relation: CarbonRelation,
     filePath: String,
-    dimensionFilePath: String,
     header: String): CarbonLoadModel = {
     val carbonLoadModel = new CarbonLoadModel
     carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
@@ -58,7 +57,6 @@ class AutoHighCardinalityIdentifyTestCase extends QueryTest with BeforeAndAfterA
     carbonLoadModel.setTableName(table.getFactTableName)
     carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
     carbonLoadModel.setFactFilePath(filePath)
-    carbonLoadModel.setDimFolderPath(dimensionFilePath)
     carbonLoadModel.setCsvHeader(header)
     carbonLoadModel.setCsvDelimiter(",")
     carbonLoadModel.setComplexDelimiterLevel1("\\$")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
index b75b1cc..4210574 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
@@ -46,7 +46,6 @@ class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAft
 
   def buildCarbonLoadModel(relation: CarbonRelation,
                            filePath: String,
-                           dimensionFilePath: String,
                            header: String): CarbonLoadModel = {
     val carbonLoadModel = new CarbonLoadModel
     carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
@@ -58,7 +57,6 @@ class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAft
     carbonLoadModel.setTableName(table.getFactTableName)
     carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
     carbonLoadModel.setFactFilePath(filePath)
-    carbonLoadModel.setDimFolderPath(dimensionFilePath)
     carbonLoadModel.setCsvHeader(header)
     carbonLoadModel.setCsvDelimiter(",")
     carbonLoadModel.setComplexDelimiterLevel1("\\$")
@@ -113,7 +111,7 @@ class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAft
     for (i <- 0 until noOfFiles) {
       val filePath: String = workDirectory + s"/src/test/resources/singlecolumn_${10 * (i
+ 1)}.csv"
       files += filePath
-      loadModels += buildCarbonLoadModel(sampleRelation, filePath, null, "empid")
+      loadModels += buildCarbonLoadModel(sampleRelation, filePath, "empid")
       writedummydata(filePath, 10 * (i + 1))
     }
     try {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
index 89aca03..a7601ee 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
@@ -50,7 +50,6 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll
{
 
   def buildCarbonLoadModel(relation: CarbonRelation,
     filePath: String,
-    dimensionFilePath: String,
     header: String): CarbonLoadModel = {
     val carbonLoadModel = new CarbonLoadModel
     carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
@@ -62,7 +61,6 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll
{
     carbonLoadModel.setTableName(table.getFactTableName)
     carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
     carbonLoadModel.setFactFilePath(filePath)
-    carbonLoadModel.setDimFolderPath(dimensionFilePath)
     carbonLoadModel.setCsvHeader(header)
     carbonLoadModel.setCsvDelimiter(",")
     carbonLoadModel.setComplexDelimiterLevel1("\\$")
@@ -156,7 +154,7 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll
{
 
   test("[issue-80]Global Dictionary Generation") {
 
-    val carbonLoadModel = buildCarbonLoadModel(sampleRelation, filePath, null, null)
+    val carbonLoadModel = buildCarbonLoadModel(sampleRelation, filePath, null)
     GlobalDictionaryUtil
       .generateGlobalDictionary(CarbonHiveContext,
         carbonLoadModel,
@@ -174,7 +172,7 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll
{
   test("[Issue-190]load csv file without header And support complex type") {
     val header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo,"
+
       "proddate,gamePointId,contractNumber"
-    val carbonLoadModel = buildCarbonLoadModel(complexRelation, complexfilePath, null, header)
+    val carbonLoadModel = buildCarbonLoadModel(complexRelation, complexfilePath, header)
     GlobalDictionaryUtil
       .generateGlobalDictionary(CarbonHiveContext,
         carbonLoadModel,
@@ -188,7 +186,6 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll
{
     // load 1
     var carbonLoadModel = buildCarbonLoadModel(incrementalLoadTableRelation,
       complexfilePath1,
-      null,
       header
     )
     GlobalDictionaryUtil
@@ -202,7 +199,6 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll
{
     // load 2
     carbonLoadModel = buildCarbonLoadModel(incrementalLoadTableRelation,
       complexfilePath2,
-      null,
       header
     )
     GlobalDictionaryUtil

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index a851d47..cafd081 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -355,12 +355,6 @@ case class LoadTable(
       carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
       carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
       carbonLoadModel.setStorePath(relation.tableMeta.storePath)
-      if (dimFilesPath.isEmpty) {
-        carbonLoadModel.setDimFolderPath(null)
-      } else {
-        val x = dimFilesPath.map(f => f.table + ":" + CarbonUtil.checkAndAppendHDFSUrl(f.loadPath))
-        carbonLoadModel.setDimFolderPath(x.mkString(","))
-      }
 
       val table = relation.tableMeta.carbonTable
       carbonLoadModel.setAggTables(table.getAggregateTablesName.asScala.toArray)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
index 09e000f..4765277 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
@@ -167,7 +167,6 @@ public class DataGraphExecuter {
             model.getCsvFilePath() != null && model.getCsvFilePath().startsWith("hdfs:");
         trans.setVariable("modifiedDimNames", model.getDimTables());
         trans.setVariable("csvInputFilePath", model.getCsvFilePath());
-        trans.setVariable("dimFileLocDir", model.getDimCSVDirLoc());
         if (hdfsReadMode) {
           trans.addParameterDefinition("vfs.hdfs.dfs.client.read.shortcircuit", "true", "");
           trans.addParameterDefinition("vfs.hdfs.dfs.domain.socket.path",
@@ -499,64 +498,6 @@ public class DataGraphExecuter {
         }
       }
     }
-
-    // Validate the Dimension CSV Files.
-    String dimFilesStr = model.getDimCSVDirLoc();
-
-    if (null != dimFilesStr && dimFilesStr.length() > 0) {
-      String[] dimMapList = model.getDimCSVDirLoc().split(",");
-
-      for (String dimFileMap : dimMapList) {
-        String tableName = dimFileMap.split(":")[0];
-        String dimCSVFileLoc = dimFileMap.substring(tableName.length() + 1);
-
-        try {
-          if (dimCSVFileLoc != null) {
-            FileType fileType = FileFactory.getFileType(dimCSVFileLoc);
-            boolean exists = FileFactory.isFileExist(dimCSVFileLoc, fileType);
-
-            if (exists) {
-              CarbonFile dimCsvFile = FileFactory.getCarbonFile(dimCSVFileLoc, fileType);
-
-              String dimFileName = dimCsvFile.getName();
-
-              if (dimFileName.endsWith(CarbonCommonConstants.CSV_FILE_EXTENSION)) {
-                String dimTableName = dimFileMap.split(":")[0];
-
-                validateDimensionCSV(schemaInfo, model.getTableName(), dimTableName, dimCsvFile,
-                    partitionId, schema, ",");
-              } else {
-                LOGGER.error(
-                    "Dimension table file provided to load Dimension tables is not a CSV
file : "
-                        + dimCSVFileLoc);
-                throw new DataLoadingException(DataProcessorConstants.CSV_VALIDATION_ERRROR_CODE,
-                    "Dimension table file provided to load Dimension tables is not a CSV
file : "
-                        + dimCSVFileLoc);
-              }
-            } else {
-              LOGGER.error(
-                  "Dimension table csv file not present in the path provided to load Dimension
"
-                      + "tables : "
-                      + dimCSVFileLoc);
-              throw new DataLoadingException(DataProcessorConstants.CSV_VALIDATION_ERRROR_CODE,
-                  "Dimension table csv file not present in the path provided to load Dimension
"
-                      + "tables : "
-                      + dimCSVFileLoc);
-            }
-          }
-        } catch (IOException e) {
-          LOGGER.error(
-              "Dimension table csv file not present in the path provided to load Dimension
tables"
-                  + " : "
-                  + dimCSVFileLoc);
-
-          throw new DataLoadingException(DataProcessorConstants.CSV_VALIDATION_ERRROR_CODE,
-              "Dimension table csv file not present in the path provided to load Dimension
tables"
-                  + " : "
-                  + dimCSVFileLoc);
-        }
-      }
-    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
index ae0853c..2900353 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
@@ -67,11 +67,6 @@ public class DataProcessTaskStatus implements IDataProcessStatus, Serializable
{
 
   private String csvFilePath;
 
-  /**
-   * dimCSVDirLoc
-   */
-  private String dimCSVDirLoc;
-
   private String dimTables;
 
   private boolean isDirectLoad;
@@ -268,14 +263,6 @@ public class DataProcessTaskStatus implements IDataProcessStatus, Serializable
{
     return copy;
   }
 
-  public String getDimCSVDirLoc() {
-    return dimCSVDirLoc;
-  }
-
-  public void setDimCSVDirLoc(String dimCSVDirLoc) {
-    this.dimCSVDirLoc = dimCSVDirLoc;
-  }
-
   public String getBlocksID() {
     return blocksID;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
index a21eb66..d071180 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
@@ -141,10 +141,6 @@ public interface IDataProcessStatus {
    */
   void setCsvFilePath(String csvFilePath);
 
-  String getDimCSVDirLoc();
-
-  void setDimCSVDirLoc(String dimCSVDirLoc);
-
   /**
    * @return the dimTables
    */

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
index 10f5197..692e5d7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
@@ -307,21 +307,6 @@ public class CarbonLoadModel implements Serializable {
     this.colDictFilePath = colDictFilePath;
   }
 
-  /**
-   * @return the dimFolderPath
-   */
-  public String getDimFolderPath() {
-    return dimFolderPath;
-  }
-
-  //TODO SIMIAN
-
-  /**
-   * @param dimFolderPath the dimFolderPath to set
-   */
-  public void setDimFolderPath(String dimFolderPath) {
-    this.dimFolderPath = dimFolderPath;
-  }
 
   /**
    * get copy with parition
@@ -332,7 +317,6 @@ public class CarbonLoadModel implements Serializable {
   public CarbonLoadModel getCopyWithPartition(String uniqueId) {
     CarbonLoadModel copy = new CarbonLoadModel();
     copy.tableName = tableName;
-    copy.dimFolderPath = dimFolderPath;
     copy.factFilePath = factFilePath + '/' + uniqueId;
     copy.databaseName = databaseName;
     copy.partitionId = uniqueId;
@@ -372,7 +356,6 @@ public class CarbonLoadModel implements Serializable {
       String header, String delimiter) {
     CarbonLoadModel copyObj = new CarbonLoadModel();
     copyObj.tableName = tableName;
-    copyObj.dimFolderPath = dimFolderPath;
     copyObj.factFilePath = null;
     copyObj.databaseName = databaseName;
     copyObj.partitionId = uniqueId;


Mime
View raw message