Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B6459200B49 for ; Tue, 19 Jul 2016 10:38:57 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id B4E15160A89; Tue, 19 Jul 2016 08:38:57 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 67EC6160A76 for ; Tue, 19 Jul 2016 10:38:56 +0200 (CEST) Received: (qmail 36991 invoked by uid 500); 19 Jul 2016 08:38:55 -0000 Mailing-List: contact commits-help@carbondata.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@carbondata.incubator.apache.org Delivered-To: mailing list commits@carbondata.incubator.apache.org Received: (qmail 36979 invoked by uid 99); 19 Jul 2016 08:38:55 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Jul 2016 08:38:55 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 2F25FC6FCB for ; Tue, 19 Jul 2016 08:38:55 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -4.507 X-Spam-Level: X-Spam-Status: No, score=-4.507 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-1.287] autolearn=disabled Received: from mx2-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id 6FQ7Brk7tXhU for ; Tue, 19 Jul 2016 08:38:52 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx2-lw-eu.apache.org (ASF Mail Server at mx2-lw-eu.apache.org) with SMTP id EA5CA60CF9 for ; Tue, 19 Jul 2016 08:38:50 +0000 (UTC) Received: (qmail 36831 invoked by uid 99); 19 Jul 2016 08:38:49 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Jul 2016 08:38:49 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 79DD2E03A6; Tue, 19 Jul 2016 08:38:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jackylk@apache.org To: commits@carbondata.incubator.apache.org Date: Tue, 19 Jul 2016 08:38:49 -0000 Message-Id: <3b954e62a2ab4e8f8491f2e7c1697719@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/5] incubator-carbondata git commit: change cube to table archived-at: Tue, 19 Jul 2016 08:38:57 -0000 Repository: incubator-carbondata Updated Branches: refs/heads/master 6948cb05d -> e5ed64bbe http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84b476bc/processing/src/main/java/org/carbondata/processing/merger/step/CarbonSliceMergerStep.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/carbondata/processing/merger/step/CarbonSliceMergerStep.java b/processing/src/main/java/org/carbondata/processing/merger/step/CarbonSliceMergerStep.java index b41b88f..6156df3 100644 --- a/processing/src/main/java/org/carbondata/processing/merger/step/CarbonSliceMergerStep.java +++ b/processing/src/main/java/org/carbondata/processing/merger/step/CarbonSliceMergerStep.java @@ -91,7 +91,8 @@ public class CarbonSliceMergerStep extends BaseStep { try { // carbon data writer step meta meta = (CarbonSliceMergerStepMeta) smi; - StandardLogService.setThreadName(StandardLogService.getPartitionID(meta.getCubeName()), null); + StandardLogService.setThreadName(StandardLogService.getPartitionID(meta.getTableName()), + null); // carbon data writer step data data = (CarbonSliceMergerStepData) sdi; @@ -128,7 +129,7 @@ public class CarbonSliceMergerStep extends BaseStep { private void renameFolders() { CarbonDataProcessorUtil.renameBadRecordsFromInProgressToNormal( - meta.getSchemaName() + File.separator + meta.getCubeName() + File.separator + meta + meta.getDatabaseName() + File.separator + meta.getTableName() + File.separator + meta .getTaskNo()); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84b476bc/processing/src/main/java/org/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java b/processing/src/main/java/org/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java index 2770453..f6dba42 100644 --- a/processing/src/main/java/org/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java +++ b/processing/src/main/java/org/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java @@ -73,14 +73,14 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta private String heirAndKeySize; /** - * schemaName + * databaseName */ - private String schemaName; + private String databaseName; /** - * cubeName + * tableName */ - private String cubeName; + private String tableName; /** * isGroupByEnabled @@ -133,8 +133,8 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta mdkeySize = ""; measureCount = ""; heirAndKeySize = ""; - cubeName = ""; - schemaName = ""; + tableName = ""; + databaseName = ""; groupByEnabled = ""; aggregatorClassString = ""; aggregatorString = ""; @@ -158,8 +158,8 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta retval.append(" ").append(XMLHandler.addTagValue("MDKeySize", mdkeySize)); retval.append(" ").append(XMLHandler.addTagValue("Measurecount", measureCount)); retval.append(" ").append(XMLHandler.addTagValue("HeirAndKeySize", heirAndKeySize)); - retval.append(" ").append(XMLHandler.addTagValue("cubeName", cubeName)); - retval.append(" ").append(XMLHandler.addTagValue("schemaName", schemaName)); + retval.append(" ").append(XMLHandler.addTagValue("tableName", tableName)); + retval.append(" ").append(XMLHandler.addTagValue("databaseName", databaseName)); retval.append(" ").append(XMLHandler.addTagValue("isGroupByEnabled", groupByEnabled)); retval.append(" ") .append(XMLHandler.addTagValue("aggregatorClassString", aggregatorClassString)); @@ -186,12 +186,12 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta @Override public void loadXML(Node stepnode, List databases, Map counters) throws KettleXMLException { try { - schemaName = XMLHandler.getTagValue(stepnode, "schemaName"); + databaseName = XMLHandler.getTagValue(stepnode, "databaseName"); tabelName = XMLHandler.getTagValue(stepnode, "TableName"); mdkeySize = XMLHandler.getTagValue(stepnode, "MDKeySize"); measureCount = XMLHandler.getTagValue(stepnode, "Measurecount"); heirAndKeySize = XMLHandler.getTagValue(stepnode, "HeirAndKeySize"); - cubeName = XMLHandler.getTagValue(stepnode, "cubeName"); + tableName = XMLHandler.getTagValue(stepnode, "tableName"); groupByEnabled = XMLHandler.getTagValue(stepnode, "isGroupByEnabled"); aggregatorClassString = XMLHandler.getTagValue(stepnode, "aggregatorClassString"); aggregatorString = XMLHandler.getTagValue(stepnode, "aggregatorString"); @@ -223,8 +223,8 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta rep.saveStepAttribute(idTransformation, idStep, "Measurecount", measureCount); rep.saveStepAttribute(idTransformation, idStep, "HeirAndKeySize", heirAndKeySize); //$NON-NLS-1$ - rep.saveStepAttribute(idTransformation, idStep, "cubeName", cubeName); //$NON-NLS-1$ - rep.saveStepAttribute(idTransformation, idStep, "schemaName", schemaName); //$NON-NLS-1$ + rep.saveStepAttribute(idTransformation, idStep, "tableName", tableName); //$NON-NLS-1$ + rep.saveStepAttribute(idTransformation, idStep, "databaseName", databaseName); //$NON-NLS-1$ rep.saveStepAttribute(idTransformation, idStep, "isGroupByEnabled", groupByEnabled); rep.saveStepAttribute(idTransformation, idStep, "aggregatorClassString", aggregatorClassString); @@ -271,8 +271,8 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta mdkeySize = rep.getStepAttributeString(idStep, "MDKeySize"); measureCount = rep.getStepAttributeString(idStep, "Measurecount"); heirAndKeySize = rep.getStepAttributeString(idStep, "HeirAndKeySize"); - schemaName = rep.getStepAttributeString(idStep, "schemaName"); - cubeName = rep.getStepAttributeString(idStep, "cubeName"); + databaseName = rep.getStepAttributeString(idStep, "databaseName"); + tableName = rep.getStepAttributeString(idStep, "tableName"); groupByEnabled = rep.getStepAttributeString(idStep, "isGroupByEnabled"); aggregatorClassString = rep.getStepAttributeString(idStep, "aggregatorClassString"); aggregatorString = rep.getStepAttributeString(idStep, "aggregatorString"); @@ -421,31 +421,31 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta } /** - * @return the schemaName + * @return the databaseName */ - public String getSchemaName() { - return schemaName; + public String getDatabaseName() { + return databaseName; } /** - * @param schemaName the schemaName to set + * @param databaseName the databaseName to set */ - public void setSchemaName(String schemaName) { - this.schemaName = schemaName; + public void setDatabaseName(String databaseName) { + this.databaseName = databaseName; } /** - * @return the cubeName + * @return the tableName */ - public String getCubeName() { - return cubeName; + public String getTableName() { + return tableName; } /** - * @param cubeName the cubeName to set + * @param tableName the tableName to set */ - public void setCubeName(String cubeName) { - this.cubeName = cubeName; + public void setTableName(String tableName) { + this.tableName = tableName; } /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84b476bc/processing/src/main/java/org/carbondata/processing/schema/metadata/ColumnsInfo.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/carbondata/processing/schema/metadata/ColumnsInfo.java b/processing/src/main/java/org/carbondata/processing/schema/metadata/ColumnsInfo.java index 9613a86..8cf206a 100644 --- a/processing/src/main/java/org/carbondata/processing/schema/metadata/ColumnsInfo.java +++ b/processing/src/main/java/org/carbondata/processing/schema/metadata/ColumnsInfo.java @@ -39,11 +39,6 @@ public class ColumnsInfo { private Map keyGenerators; /** - * Cube name - */ - private String cubeName; - - /** * Hierarchy table names (Same will be file names for file store or * incremental load) */ @@ -88,7 +83,7 @@ public class ColumnsInfo { private int[][] propIndx; /** - * Dimension Index, Property column names from cube. + * Dimension Index, Property column names from table. * [0] - [col2,col3] [1] - [col4,col7,col8] [2] - [] */ private List[] propColumns; @@ -140,7 +135,7 @@ public class ColumnsInfo { private boolean[] dimsPresent; - private String schemaName; + private String databaseName; private Map complexTypesMap; @@ -193,17 +188,17 @@ public class ColumnsInfo { } /** - * @return Returns the cubeName. + * @return Returns the tableName. */ - public String getCubeName() { - return cubeName; + public String getTableName() { + return tableName; } /** - * @param cubeName The cubeName to set. + * @param tableName The tableName to set. */ - public void setCubeName(String cubeName) { - this.cubeName = cubeName; + public void setTableName(String tableName) { + this.tableName = tableName; } /** @@ -414,20 +409,6 @@ public class ColumnsInfo { } /** - * @return Returns the tableName. - */ - public String getTableName() { - return tableName; - } - - /** - * @param tableName The tableName to set. - */ - public void setTableName(String tableName) { - this.tableName = tableName; - } - - /** * @return Returns the primaryKeyMap. */ public Map getPrimaryKeyMap() { @@ -473,12 +454,12 @@ public class ColumnsInfo { this.measureColumns = measureColumns; } - public String getSchemaName() { - return schemaName; + public String getDatabaseName() { + return databaseName; } - public void setSchemaName(String schemaName) { - this.schemaName = schemaName; + public void setDatabaseName(String databaseName) { + this.databaseName = databaseName; } /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84b476bc/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java b/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java index 0f68ff3..ebba07a 100644 --- a/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java +++ b/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java @@ -67,10 +67,6 @@ public class SortDataRows { */ private int entryCount; /** - * tableName - */ - private String tableName; - /** * sortBufferSize */ private int sortBufferSize; @@ -134,8 +130,8 @@ public class SortDataRows { * bufferSize */ private int bufferSize; - private String schemaName; - private String cubeName; + private String databaseName; + private String tableName; private char[] aggType; @@ -200,10 +196,10 @@ public class SortDataRows { /** * This method will be used to initialize */ - public void initialize(String schemaName, String cubeName) + public void initialize(String databaseName, String tableName) throws CarbonSortKeyAndGroupByException { - this.schemaName = schemaName; - this.cubeName = cubeName; + this.databaseName = databaseName; + this.tableName = tableName; CarbonProperties carbonProperties = CarbonProperties.getInstance(); setSortConfiguration(carbonProperties); @@ -276,7 +272,7 @@ public class SortDataRows { private void initAggType() { Arrays.fill(aggType, 'n'); CarbonTable carbonTable = CarbonMetadata.getInstance() - .getCarbonTable(schemaName + CarbonCommonConstants.UNDERSCORE + tableName); + .getCarbonTable(databaseName + CarbonCommonConstants.UNDERSCORE + tableName); List measures = carbonTable.getMeasureByTableName(tableName); for (int i = 0; i < measureColCount; i++) { aggType[i] = DataTypeUtil.getAggType(measures.get(i).getDataType()); @@ -502,7 +498,7 @@ public class SortDataRows { // get sort buffer size this.sortBufferSize = Integer.parseInt(instance .getProperty(CarbonCommonConstants.SORT_SIZE, CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL)); - LOGGER.info("Sort size for cube: " + this.sortBufferSize); + LOGGER.info("Sort size for table: " + this.sortBufferSize); // set number of intermedaite file to merge this.numberOfIntermediateFileToBeMerged = Integer.parseInt(instance .getProperty(CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT, @@ -525,7 +521,7 @@ public class SortDataRows { */ private void updateSortTempFileLocation() { String carbonDataDirectoryPath = CarbonDataProcessorUtil - .getLocalDataFolderLocation(schemaName, tableName, taskNo, partitionID, + .getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId); this.tempFileLocation = carbonDataDirectoryPath + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84b476bc/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStep.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStep.java b/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStep.java index c7b847d..199db5a 100644 --- a/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStep.java +++ b/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStep.java @@ -175,7 +175,7 @@ public class SortKeyStep extends BaseStep { meta.getTaskNo(), this.noDictionaryColMaping); try { // initialize sort - this.sortDataRows.initialize(meta.getSchemaName(), meta.getCubeName()); + this.sortDataRows.initialize(meta.getDatabaseName(), meta.getTableName()); } catch (CarbonSortKeyAndGroupByException e) { throw new KettleException(e); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84b476bc/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java b/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java index afbe932..57da6cb 100644 --- a/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java +++ b/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java @@ -60,14 +60,14 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface { private String outputRowSize; /** - * cubeName + * tableName */ - private String cubeName; + private String tableName; /** - * schemaName + * databaseName */ - private String schemaName; + private String databaseName; /** * Dimension Count @@ -125,10 +125,10 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface { this.tabelName = ""; factDimLensString = ""; outputRowSize = ""; - schemaName = ""; + databaseName = ""; noDictionaryDims = ""; noDictionaryDimsMapping = ""; - cubeName = ""; + tableName = ""; dimensionCount = ""; complexDimensionCount = ""; measureCount = ""; @@ -151,8 +151,8 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface { retval.append(" ").append(XMLHandler.addTagValue("TableName", this.tabelName)); retval.append(" ").append(XMLHandler.addTagValue("factDimLensString", factDimLensString)); retval.append(" ").append(XMLHandler.addTagValue("outputRowSize", this.outputRowSize)); - retval.append(" ").append(XMLHandler.addTagValue("cubeName", this.cubeName)); - retval.append(" ").append(XMLHandler.addTagValue("schemaName", this.schemaName)); + retval.append(" ").append(XMLHandler.addTagValue("tableName", this.tableName)); + retval.append(" ").append(XMLHandler.addTagValue("databaseName", this.databaseName)); retval.append(" ").append(XMLHandler.addTagValue("dimensionCount", this.dimensionCount)); retval.append(" ").append(XMLHandler.addTagValue("noDictionaryDims", this.noDictionaryDims)); retval.append(" ") @@ -185,8 +185,8 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface { this.tabelName = XMLHandler.getTagValue(stepnode, "TableName"); this.outputRowSize = XMLHandler.getTagValue(stepnode, "outputRowSize"); this.factDimLensString = XMLHandler.getTagValue(stepnode, "factDimLensString"); - this.cubeName = XMLHandler.getTagValue(stepnode, "cubeName"); - this.schemaName = XMLHandler.getTagValue(stepnode, "schemaName"); + this.tableName = XMLHandler.getTagValue(stepnode, "tableName"); + this.databaseName = XMLHandler.getTagValue(stepnode, "databaseName"); this.dimensionCount = XMLHandler.getTagValue(stepnode, "dimensionCount"); this.noDictionaryDims = XMLHandler.getTagValue(stepnode, "noDictionaryDims"); this.noDictionaryDimsMapping = XMLHandler.getTagValue(stepnode, "noDictionaryDimsMapping"); @@ -219,8 +219,8 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface { rep.saveStepAttribute(idTransformation, idStep, "factDimLensString", factDimLensString); rep.saveStepAttribute(idTransformation, idStep, "outputRowSize", this.outputRowSize); - rep.saveStepAttribute(idTransformation, idStep, "cubeName", this.cubeName); - rep.saveStepAttribute(idTransformation, idStep, "schemaName", this.schemaName); + rep.saveStepAttribute(idTransformation, idStep, "tableName", this.tableName); + rep.saveStepAttribute(idTransformation, idStep, "databaseName", this.databaseName); rep.saveStepAttribute(idTransformation, idStep, "dimensionCount", this.dimensionCount); rep.saveStepAttribute(idTransformation, idStep, "noDictionaryDims", this.noDictionaryDims); rep.saveStepAttribute(idTransformation, idStep, "noDictionaryDimsMapping", @@ -257,8 +257,8 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface { try { this.tabelName = rep.getStepAttributeString(idStep, "TableName"); this.outputRowSize = rep.getStepAttributeString(idStep, "outputRowSize"); - this.schemaName = rep.getStepAttributeString(idStep, "schemaName"); - this.cubeName = rep.getStepAttributeString(idStep, "cubeName"); + this.databaseName = rep.getStepAttributeString(idStep, "databaseName"); + this.tableName = rep.getStepAttributeString(idStep, "tableName"); this.dimensionCount = rep.getStepAttributeString(idStep, "dimensionCount"); this.noDictionaryDims = rep.getStepAttributeString(idStep, "noDictionaryDims"); this.noDictionaryDims = rep.getStepAttributeString(idStep, "noDictionaryDimsMapping"); @@ -358,31 +358,31 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface { } /** - * @return the cubeName + * @return the tableName */ - public String getCubeName() { - return cubeName; + public String getTableName() { + return tableName; } /** - * @param cubeName the cubeName to set + * @param tableName the tableName to set */ - public void setCubeName(String cubeName) { - this.cubeName = cubeName; + public void setTableName(String tableName) { + this.tableName = tableName; } /** - * @return the schemaName + * @return the databaseName */ - public String getSchemaName() { - return schemaName; + public String getDatabaseName() { + return databaseName; } /** - * @param schemaName the schemaName to set + * @param databaseName the databaseName to set */ - public void setSchemaName(String schemaName) { - this.schemaName = schemaName; + public void setDatabaseName(String databaseName) { + this.databaseName = databaseName; } /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84b476bc/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordslogger.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordslogger.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordslogger.java index 4cd79ec..8e56a9b 100644 --- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordslogger.java +++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordslogger.java @@ -67,7 +67,7 @@ public class BadRecordslogger { */ private CarbonFile logFile; /** - * task key which is Schemaname/CubeName/tablename + * task key which is DatabaseName/TableName/tablename */ private String taskKey; @@ -81,7 +81,7 @@ public class BadRecordslogger { } /** - * @param key Schemaname/CubeName/tablename + * @param key DatabaseName/TableName/tablename * @return return "Partially" and remove from map */ public static String hasBadRecord(String key) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84b476bc/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java index 77f7297..4a85f25 100644 --- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java +++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java @@ -343,14 +343,11 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn * denormColumNames */ private String denormColumNames; + /** - * cubeName - */ - private String cubeName; - /** - * schemaName + * databaseName */ - private String schemaName; + private String databaseName; private int currentRestructNumber; /** * partitionID @@ -699,8 +696,8 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn retval.append(" ").append(XMLHandler.addTagValue("measureDataType", measureDataType)); retval.append(" ").append(XMLHandler.addTagValue("columnAndTableName_ColumnMapForAggString", columnAndTableNameColumnMapForAggString)); - retval.append(" ").append(XMLHandler.addTagValue("schemaName", schemaName)); - retval.append(" ").append(XMLHandler.addTagValue("cubeName", cubeName)); + retval.append(" ").append(XMLHandler.addTagValue("databaseName", databaseName)); + retval.append(" ").append(XMLHandler.addTagValue("tableName", tableName)); retval.append(" ").append(XMLHandler.addTagValue("denormColumNames", denormColumNames)); retval.append(" ") .append(XMLHandler.addTagValue("currentRestructNumber", currentRestructNumber)); @@ -749,8 +746,8 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn noDictionaryDims = XMLHandler.getTagValue(stepnode, "dimNoDictionary"); columnsDataTypeString = XMLHandler.getTagValue(stepnode, "dimColDataTypes"); tableName = XMLHandler.getTagValue(stepnode, "factOrAggTable"); - cubeName = XMLHandler.getTagValue(stepnode, "cubeName"); - schemaName = XMLHandler.getTagValue(stepnode, "schemaName"); + tableName = XMLHandler.getTagValue(stepnode, "tableName"); + databaseName = XMLHandler.getTagValue(stepnode, "databaseName"); denormColumNames = XMLHandler.getTagValue(stepnode, "denormColumNames"); currentRestructNumber = Integer.parseInt(XMLHandler.getTagValue(stepnode, "currentRestructNumber")); @@ -1315,9 +1312,9 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn measureDataType = rep.getStepAttributeString(idStep, "measureDataType"); columnAndTableNameColumnMapForAggString = rep.getStepAttributeString(idStep, "columnAndTableName_ColumnMapForAggString"); - schemaName = rep.getStepAttributeString(idStep, "schemaName"); + databaseName = rep.getStepAttributeString(idStep, "databaseName"); - cubeName = rep.getStepAttributeString(idStep, "cubeName"); + tableName = rep.getStepAttributeString(idStep, "tableName"); denormColumNames = rep.getStepAttributeString(idStep, "denormColumNames"); currentRestructNumber = (int) rep.getStepAttributeInteger(idStep, "currentRestructNumber"); partitionID = rep.getStepAttributeString(idStep, "partitionID"); @@ -1369,8 +1366,8 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn rep.saveStepAttribute(idTransformation, idStep, "heirNadDimsLensString", heirNadDimsLensString); rep.saveStepAttribute(idTransformation, idStep, "measureDataType", measureDataType); - rep.saveStepAttribute(idTransformation, idStep, "schemaName", schemaName); - rep.saveStepAttribute(idTransformation, idStep, "cubeName", cubeName); + rep.saveStepAttribute(idTransformation, idStep, "databaseName", databaseName); + rep.saveStepAttribute(idTransformation, idStep, "tableName", tableName); rep.saveStepAttribute(idTransformation, idStep, "denormColumNames", denormColumNames); rep.saveStepAttribute(idTransformation, idStep, "currentRestructNumber", currentRestructNumber); @@ -1556,20 +1553,12 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn this.columnAndTableNameColumnMapForAggString = columnAndTableNameColumnMapForAggString; } - public String getCubeName() { - return cubeName; - } - - public void setCubeName(String cubeName) { - this.cubeName = cubeName; - } - - public String getSchemaName() { - return schemaName; + public String getDatabaseName() { + return databaseName; } - public void setSchemaName(String schemaName) { - this.schemaName = schemaName; + public void setDatabaseName(String databaseName) { + this.databaseName = databaseName; } public String getDenormColumNames() { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84b476bc/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java index d652a65..15c5660 100644 --- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java +++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java @@ -293,7 +293,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { first = false; meta.initialize(); final Object dataProcessingLockObject = CarbonDataProcessorManager.getInstance() - .getDataProcessingLockObject(meta.getSchemaName() + '_' + meta.getCubeName()); + .getDataProcessingLockObject(meta.getDatabaseName() + '_' + meta.getTableName()); synchronized (dataProcessingLockObject) { // observer of writing file in thread this.threadStatusObserver = new ThreadStatusObserver(); @@ -347,8 +347,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { columnsInfo.setDims(meta.dims); columnsInfo.setDimColNames(meta.dimColNames); columnsInfo.setKeyGenerators(data.getKeyGenerators()); - columnsInfo.setSchemaName(meta.getSchemaName()); - columnsInfo.setCubeName(meta.getCubeName()); + columnsInfo.setDatabaseName(meta.getDatabaseName()); + columnsInfo.setTableName(meta.getTableName()); columnsInfo.setHierTables(meta.hirches.keySet()); columnsInfo.setBatchSize(meta.getBatchSize()); columnsInfo.setStoreType(meta.getStoreType()); @@ -369,9 +369,10 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { columnsInfo.setColumnSchemaDetailsWrapper(meta.getColumnSchemaDetailsWrapper()); columnsInfo.setColumnProperties(meta.getColumnPropertiesMap()); updateBagLogFileName(); - String key = meta.getSchemaName() + '/' + meta.getCubeName() + '_' + meta.getTableName(); + String key = meta.getDatabaseName() + '/' + meta.getTableName() + + '_' + meta.getTableName(); badRecordslogger = new BadRecordslogger(key, csvFilepath, getBadLogStoreLocation( - meta.getSchemaName() + '/' + meta.getCubeName() + "/" + meta.getTaskNo())); + meta.getDatabaseName() + '/' + meta.getTableName() + "/" + meta.getTaskNo())); columnsInfo.setTimeOrdinalIndices(meta.timeOrdinalIndices); surrogateKeyGen = new FileStoreSurrogateKeyGenForCSV(columnsInfo, meta.getPartitionID(), @@ -696,7 +697,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { */ private void updateStoreLocation() { loadFolderLoc = CarbonDataProcessorUtil - .getLocalDataFolderLocation(meta.getSchemaName(), meta.getTableName(), meta.getTaskNo(), + .getLocalDataFolderLocation(meta.getDatabaseName(), meta.getTableName(), meta.getTaskNo(), meta.getPartitionID(), meta.getSegmentId()+""); } @@ -724,7 +725,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { Callable callable = new Callable() { @Override public Void call() throws RuntimeException { StandardLogService - .setThreadName(StandardLogService.getPartitionID(meta.getCubeName()), null); + .setThreadName(StandardLogService.getPartitionID(meta.getTableName()), null); try { doProcess(); } catch (Throwable e) { @@ -843,7 +844,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { private String getCarbonLocalBaseStoreLocation() { String tempLocationKey = - meta.getSchemaName() + CarbonCommonConstants.UNDERSCORE + meta.getCubeName() + meta.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + meta.getTableName() + CarbonCommonConstants.UNDERSCORE + meta.getTaskNo(); String strLoc = CarbonProperties.getInstance() .getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL); @@ -1805,7 +1806,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { */ private void populateCarbonMeasures(String[] measures) { CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable( - meta.getSchemaName() + CarbonCommonConstants.UNDERSCORE + meta.getTableName()); + meta.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + meta.getTableName()); meta.carbonMeasures = new CarbonMeasure[measures.length]; msrDataType = new DataType[measures.length]; for (int i = 0; i < measures.length; i++) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84b476bc/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java index 5f7be3c..a1182b3 100644 --- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java +++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java @@ -122,8 +122,8 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe baseStorePath = columnsInfo.getBaseStoreLocation(); setStoreFolderWithLoadNumber( - checkAndCreateLoadFolderNumber(columnsInfo.getSchemaName(), - columnsInfo.getCubeName())); + checkAndCreateLoadFolderNumber(columnsInfo.getDatabaseName(), + columnsInfo.getTableName())); fileManager = new FileManager(); fileManager.setName(loadFolderName + CarbonCommonConstants.FILE_INPROGRESS_STATUS); @@ -217,7 +217,7 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS); String[] dimColumnNames = columnsInfo.getDimColNames(); String[] dimColumnIds = columnsInfo.getDimensionColumnIds(); - String databaseName = columnsInfo.getSchemaName(); + String databaseName = columnsInfo.getDatabaseName(); String tableName = columnsInfo.getTableName(); CarbonTable carbonTable = CarbonMetadata.getInstance() .getCarbonTable(databaseName + CarbonCommonConstants.UNDERSCORE + tableName); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/84b476bc/processing/src/main/java/org/carbondata/processing/util/CarbonSchemaParser.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/carbondata/processing/util/CarbonSchemaParser.java b/processing/src/main/java/org/carbondata/processing/util/CarbonSchemaParser.java index 3a789f1..4a40809 100644 --- a/processing/src/main/java/org/carbondata/processing/util/CarbonSchemaParser.java +++ b/processing/src/main/java/org/carbondata/processing/util/CarbonSchemaParser.java @@ -95,12 +95,6 @@ public final class CarbonSchemaParser { return columnstr; } - /** - * @param cube - * @param dimensions - * @param isQuotesRequired - * @return - */ private static String getDimensionSQLQueries(List dimensions, CarbonDataLoadSchema carbonDataLoadSchema) { // @@ -148,7 +142,7 @@ public final class CarbonSchemaParser { } /** - * @param cube + * @param table * @param dimensions * @return */ @@ -344,10 +338,10 @@ public final class CarbonSchemaParser { } /** - * Get dimension string from a array of CubeDimension,which can be shared - * CubeDimension within schema or in a cube. + * Get dimension string from a array of TableDimension,which can be shared + * TableDimension within schema or in a table. * - * @param cube + * @param table * @param dimensions * @return */ @@ -535,7 +529,7 @@ public final class CarbonSchemaParser { * @param carbonDataLoadSchema * @return */ - public static String[] getCubeDimensions(List dimensions, + public static String[] getTableDimensions(List dimensions, CarbonDataLoadSchema carbonDataLoadSchema) { List list = new ArrayList(CarbonCommonConstants.CONSTANT_SIZE_TEN); for (CarbonDimension cDimension : dimensions) { @@ -941,7 +935,7 @@ public final class CarbonSchemaParser { /** * Get Measure Name String * - * @param cube + * @param table * @return */ public static String getMeasuresNamesString(List measures) { @@ -966,7 +960,7 @@ public final class CarbonSchemaParser { /** * Get Measure Name String * - * @param cube + * @param table * @return */ public static String getMeasuresUniqueColumnNamesString(List measures) { @@ -991,7 +985,7 @@ public final class CarbonSchemaParser { /** * Get Measure Name String * - * @param cube + * @param table * @return */ public static String getMeasuresNamesStringForAgg(String[] measures) { @@ -1015,7 +1009,7 @@ public final class CarbonSchemaParser { /** * Get Measure Aggregator array * - * @param cube + * @param table * @return */ public static String[] getMeasuresAggragatorArray(List measures) { @@ -1030,7 +1024,7 @@ public final class CarbonSchemaParser { /** * @param schemaInfo - * @param cube + * @param table * @return */ public static String getActualDimensions(List dimensions) { @@ -1057,7 +1051,7 @@ public final class CarbonSchemaParser { /**Method will prepare column name and its data type string inorder * to pass to the ETL steps. * @param schemaInfo - * @param cube + * @param table * @return */ public static String getDimensionsDataTypes(List dimensions) { @@ -1073,7 +1067,7 @@ public final class CarbonSchemaParser { } /** - * @param cube + * @param table * @return */ public static String getActualDimensionsForAggregate(String[] columns) { @@ -1119,7 +1113,7 @@ public final class CarbonSchemaParser { * * @param dimensions * @param schema - * @param cube + * @param table * @return String */ public static String getLevelAndDataTypeMapString(List dimensions, @@ -1140,7 +1134,7 @@ public final class CarbonSchemaParser { * * @param dimensions * @param schema - * @param cube + * @param table * @return String */ public static String getComplexTypeString(List dimensions) { @@ -1206,10 +1200,10 @@ public final class CarbonSchemaParser { } /** - * Get dimension string from a array of CubeDimension,which can be shared - * CubeDimension within schema or in a cube. + * Get dimension string from a array of TableDimension,which can be shared + * TableDimension within schema or in a table. * - * @param cube + * @param table * @param dimensions * @return */