carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [07/23] carbondata git commit: [CARBONDATA-1482] fixed the failing test cases of presto integration
Date Thu, 21 Sep 2017 00:27:17 GMT
[CARBONDATA-1482] fixed the failing test cases of presto integration

1.builds are passing
2.added dependency for spark guava
3.throw exception in case carbondatastore creator is unable to load the data
4.fixed the timestamp issue due to which test cases are failing

This closes #1360


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d60d973d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d60d973d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d60d973d

Branch: refs/heads/streaming_ingest
Commit: d60d973dfbb69c1b740a6a00cab878bc168c14ae
Parents: 6f20437
Author: anubhav100 <anubhav.tarar@knoldus.in>
Authored: Fri Sep 15 15:12:10 2017 +0530
Committer: Ravindra Pesala <ravi.pesala@gmail.com>
Committed: Fri Sep 15 19:25:10 2017 +0530

----------------------------------------------------------------------
 integration/presto/pom.xml                      |  7 ++++
 .../integrationtest/PrestoAllDataTypeTest.scala | 26 +++++++-------
 .../presto/util/CarbonDataStoreCreator.scala    | 36 ++++++++++++--------
 3 files changed, 42 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/d60d973d/integration/presto/pom.xml
----------------------------------------------------------------------
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index b23b1be..50e6349 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -438,6 +438,13 @@
         </exclusion>
       </exclusions>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-network-common_2.11</artifactId>
+      <scope>test</scope>
+      <version>2.1.0</version>
+    </dependency>
   </dependencies>
 
   <build>

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d60d973d/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
index 1743be6..433ddd4 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
@@ -14,12 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.carbondata.presto.integrationtest
 
 import java.io.File
 
 import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}
-import util.CarbonDataStoreCreator
 
 import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.presto.server.PrestoServer
@@ -35,8 +35,10 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll
{
   private val storePath = s"$rootPath/integration/presto/target/store"
 
   override def beforeAll: Unit = {
+    import org.apache.carbondata.presto.util.CarbonDataStoreCreator
     CarbonDataStoreCreator
-      .createCarbonStore(storePath, s"$rootPath/integration/presto/src/test/resources/alldatatype.csv")
+      .createCarbonStore(storePath,
+        s"$rootPath/integration/presto/src/test/resources/alldatatype.csv")
     logger.info(s"\nCarbon store is created at location: $storePath")
     PrestoServer.startServer(storePath)
   }
@@ -91,21 +93,21 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll
{
   test("test the result for count()clause with distinct operator on decimal column in presto")
{
     val actualResult: List[Map[String, Any]] = PrestoServer
       .executeQuery("SELECT COUNT(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
-    val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 7))
-    actualResult.equals(expectedResult)
+    val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 10))
+    assert(actualResult.equals(expectedResult))
   }
   test("test the result for count()clause with out  distinct operator on decimal column in
presto")
   {
     val actualResult: List[Map[String, Any]] = PrestoServer
       .executeQuery("SELECT COUNT(BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
     val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 10))
-    actualResult.equals(expectedResult)
+    assert(actualResult.equals(expectedResult))
   }
   test("test the result for sum()with out distinct operator for decimal column in presto")
{
     val actualResult: List[Map[String, Any]] = PrestoServer
       .executeQuery("SELECT SUM(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
-    val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 54))
-    actualResult.equals(expectedResult)
+    val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 20774.6475))
+    assert(actualResult.toString().equals(expectedResult.toString()))
   }
   test("test the result for sum() with distinct operator for decimal column in presto") {
     val actualResult: List[Map[String, Any]] = PrestoServer
@@ -115,11 +117,11 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll
{
       actualResult.head("RESULT").toString.toDouble ==
       expectedResult.head("RESULT").toString.toDouble)
   }
-  test("test the result for avg() with distinct operator on decimal coin presto") {
+  test("test the result for avg() with distinct operator on decimal on presto") {
     val actualResult: List[Map[String, Any]] = PrestoServer
       .executeQuery("SELECT AVG(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
-    val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 8900))
-    actualResult.equals(expectedResult)
+    val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 2077.4648))
+    assert(actualResult.toString.equals(expectedResult.toString))
   }
 
   test("test the result for min() with distinct operator in decimalType of presto") {
@@ -127,7 +129,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll
{
       .executeQuery("SELECT MIN(BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
     val expectedResult: List[Map[String, Any]] = List(Map(
       "RESULT" -> java.math.BigDecimal.valueOf(500.414).setScale(4)))
-    actualResult.equals(expectedResult)
+    assert(actualResult.equals(expectedResult))
   }
 
   test("test the result for max() with distinct operator in decimalType of presto") {
@@ -135,7 +137,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll
{
       .executeQuery("SELECT MAX(BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
     val expectedResult: List[Map[String, Any]] = List(Map(
       "RESULT" -> java.math.BigDecimal.valueOf(9999.999).setScale(4)))
-    actualResult.equals(expectedResult)
+    assert(actualResult.equals(expectedResult))
   }
   test("select decimal data type with ORDER BY  clause") {
     val actualResult: List[Map[String, Any]] = PrestoServer

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d60d973d/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index 6cb97f1..64f892d 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -15,12 +15,12 @@
  * limitations under the License.
  */
 
-package util
+package org.apache.carbondata.presto.util
 
+import java.util
 import java.io._
 import java.nio.charset.Charset
 import java.text.SimpleDateFormat
-import java.util
 import java.util.{ArrayList, Date, List, UUID}
 
 import scala.collection.JavaConversions._
@@ -30,34 +30,34 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.io.NullWritable
 import org.apache.hadoop.mapred.TaskAttemptID
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
 import org.apache.hadoop.mapreduce.{RecordReader, TaskType}
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
 
 import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
 import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier,
 ReverseDictionary}
-import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.datastore.impl.FileFactory
 import org.apache.carbondata.core.fileoperations.{AtomicFileOperations, AtomicFileOperationsImpl,
 FileWriteOperation}
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata,
+CarbonTableIdentifier, ColumnIdentifier}
 import org.apache.carbondata.core.metadata.converter.{SchemaConverter,
 ThriftWrapperSchemaConverterImpl}
 import org.apache.carbondata.core.metadata.datatype.DataType
 import org.apache.carbondata.core.metadata.encoder.Encoding
+import org.apache.carbondata.core.metadata.schema.{SchemaEvolution, SchemaEvolutionEntry}
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema}
 import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension,
 CarbonMeasure, ColumnSchema}
-import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema}
-import org.apache.carbondata.core.metadata.schema.{SchemaEvolution, SchemaEvolutionEntry}
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata,
-CarbonTableIdentifier, ColumnIdentifier}
 import org.apache.carbondata.core.statusmanager.LoadMetadataDetails
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
 import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.writer.sortindex.{CarbonDictionarySortIndexWriter,
-CarbonDictionarySortIndexWriterImpl, CarbonDictionarySortInfo, CarbonDictionarySortInfoPreparator}
+import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
 import org.apache.carbondata.core.writer.{CarbonDictionaryWriter, CarbonDictionaryWriterImpl,
 ThriftWriter}
+import org.apache.carbondata.core.writer.sortindex.{CarbonDictionarySortIndexWriter,
+CarbonDictionarySortIndexWriterImpl, CarbonDictionarySortInfo, CarbonDictionarySortInfoPreparator}
 import org.apache.carbondata.processing.api.dataloader.SchemaInfo
 import org.apache.carbondata.processing.constants.TableOptionConstant
 import org.apache.carbondata.processing.csvload.{BlockDetails, CSVInputFormat,
@@ -65,6 +65,7 @@ CSVRecordReaderIterator, StringArrayWritable}
 import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
 import org.apache.carbondata.processing.newflow.DataLoadExecutor
 import org.apache.carbondata.processing.newflow.constants.DataLoadProcessorConstants
+import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException
 
 object CarbonDataStoreCreator {
 
@@ -110,7 +111,7 @@ object CarbonDataStoreCreator {
       loadModel.setDefaultTimestampFormat(
         CarbonProperties.getInstance.getProperty(
           CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
-          CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
+          CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS))
       loadModel.setDefaultDateFormat(
         CarbonProperties.getInstance.getProperty(
           CarbonCommonConstants.CARBON_DATE_FORMAT,
@@ -397,7 +398,11 @@ object CarbonDataStoreCreator {
           dictionarySortInfo.getSortIndexInverted)
       }
       catch {
-        case exception: Exception => logger.error(s"exception occurs $exception")
+        case exception: Exception =>
+
+
+          logger.error(s"exception occurs $exception")
+          throw new CarbonDataLoadingException("Data Loading Failed")
       }
       finally carbonDictionaryWriter.close()
     }
@@ -545,13 +550,14 @@ object CarbonDataStoreCreator {
       writeOperation.close()
     }
     catch {
-      case exception: Exception => logger.error(s"Exception occurs $exception")
+      case exception: Exception => logger.error(s"exception occurs $exception")
+        throw new CarbonDataLoadingException("Data Loading Failed")
     }
   }
 
   private def readCurrentTime(): String = {
     val sdf: SimpleDateFormat = new SimpleDateFormat(
-      CarbonCommonConstants.CARBON_TIMESTAMP)
+      CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS)
     sdf.format(new Date())
   }
 


Mime
View raw message