carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [24/45] carbondata git commit: [CARBONDATA-2952] Provide CarbonReader C++ interface for SDK
Date Tue, 09 Oct 2018 15:50:26 GMT
[CARBONDATA-2952] Provide CarbonReader C++ interface for SDK

1.init carbonreader,config data path and tablename
2.config ak sk endpoing for S3
3.configure projection
4.build carbon reader
5.hasNext
6.readNextRow
7.close

This closes #2738


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e9a198ab
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e9a198ab
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e9a198ab

Branch: refs/heads/branch-1.5
Commit: e9a198ab65c7bc9642e8a3186072f39aa2e5f0f6
Parents: d8a51c9
Author: xubo245 <xubo29@huawei.com>
Authored: Thu Sep 20 18:35:34 2018 +0800
Committer: kunal642 <kunalkapoor642@gmail.com>
Committed: Wed Oct 3 17:53:35 2018 +0530

----------------------------------------------------------------------
 .gitignore                                      |   1 +
 .../core/constants/CarbonCommonConstants.java   |   4 +
 docs/CSDK-guide.md                              | 197 ++++++++++++++++++
 examples/spark2/pom.xml                         |   2 +-
 pom.xml                                         |   2 +
 store/CSDK/CMakeLists.txt                       |  17 ++
 store/CSDK/CarbonReader.cpp                     | 103 ++++++++++
 store/CSDK/CarbonReader.h                       | 105 ++++++++++
 store/CSDK/main.cpp                             | 200 +++++++++++++++++++
 store/sdk/pom.xml                               |   5 +
 .../carbondata/sdk/file/CarbonReader.java       |  30 +++
 .../sdk/file/CarbonReaderBuilder.java           |  16 ++
 12 files changed, 681 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 5d66a40..00e4934 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@
 .settings
 .cache
 target/
+store/CSDK/cmake-build-debug/*
 .project
 .classpath
 metastore_db/

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index faad0dc..46139c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -507,6 +507,10 @@ public final class CarbonCommonConstants {
    */
   public static final String FILE_SEPARATOR = "/";
   /**
+   * ARRAY separator
+   */
+  public static final String ARRAY_SEPARATOR = "\001";
+  /**
    * MAX_QUERY_EXECUTION_TIME
    */
   @CarbonProperty

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/docs/CSDK-guide.md
----------------------------------------------------------------------
diff --git a/docs/CSDK-guide.md b/docs/CSDK-guide.md
new file mode 100644
index 0000000..c4f4a31
--- /dev/null
+++ b/docs/CSDK-guide.md
@@ -0,0 +1,197 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more 
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership. 
+    The ASF licenses this file to you under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with 
+    the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+    
+    Unless required by applicable law or agreed to in writing, software 
+    distributed under the License is distributed on an "AS IS" BASIS, 
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and 
+    limitations under the License.
+-->
+
+# CSDK Guide
+
+CarbonData CSDK provides C++ interface to write and read carbon file. 
+CSDK use JNI to invoke java SDK in C++ code.
+
+
+# CSDK Reader
+This CSDK reader reads CarbonData file and carbonindex file at a given path.
+External client can make use of this reader to read CarbonData files in C++ 
+code and without CarbonSession.
+
+
+In the carbon jars package, there exist a carbondata-sdk.jar, 
+including SDK reader for CSDK.
+## Quick example
+```
+// 1. init JVM
+JavaVM *jvm;
+JNIEnv *initJVM() {
+    JNIEnv *env;
+    JavaVMInitArgs vm_args;
+    int parNum = 3;
+    int res;
+    JavaVMOption options[parNum];
+
+    options[0].optionString = "-Djava.compiler=NONE";
+    options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar";
+    options[2].optionString = "-verbose:jni";
+    vm_args.version = JNI_VERSION_1_8;
+    vm_args.nOptions = parNum;
+    vm_args.options = options;
+    vm_args.ignoreUnrecognized = JNI_FALSE;
+
+    res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args);
+    if (res < 0) {
+        fprintf(stderr, "\nCan't create Java VM\n");
+        exit(1);
+    }
+
+    return env;
+}
+
+// 2. create carbon reader and read data 
+// 2.1 read data from local disk
+/**
+ * test read data from local disk, without projection
+ *
+ * @param env  jni env
+ * @return
+ */
+bool readFromLocalWithoutProjection(JNIEnv *env) {
+
+    CarbonReader carbonReaderClass;
+    carbonReaderClass.builder(env, "../resources/carbondata", "test");
+    carbonReaderClass.build();
+
+    while (carbonReaderClass.hasNext()) {
+        jobjectArray row = carbonReaderClass.readNextRow();
+        jsize length = env->GetArrayLength(row);
+        int j = 0;
+        for (j = 0; j < length; j++) {
+            jobject element = env->GetObjectArrayElement(row, j);
+            char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+            printf("%s\t", str);
+        }
+        printf("\n");
+    }
+    carbonReaderClass.close();
+}
+
+// 2.2 read data from S3
+
+/**
+ * read data from S3
+ * parameter is ak sk endpoint
+ *
+ * @param env jni env
+ * @param argv argument vector
+ * @return
+ */
+bool readFromS3(JNIEnv *env, char *argv[]) {
+    CarbonReader reader;
+
+    char *args[3];
+    // "your access key"
+    args[0] = argv[1];
+    // "your secret key"
+    args[1] = argv[2];
+    // "your endPoint"
+    args[2] = argv[3];
+
+    reader.builder(env, "s3a://sdk/WriterOutput", "test");
+    reader.withHadoopConf(3, args);
+    reader.build();
+    printf("\nRead data from S3:\n");
+    while (reader.hasNext()) {
+        jobjectArray row = reader.readNextRow();
+        jsize length = env->GetArrayLength(row);
+
+        int j = 0;
+        for (j = 0; j < length; j++) {
+            jobject element = env->GetObjectArrayElement(row, j);
+            char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+            printf("%s\t", str);
+        }
+        printf("\n");
+    }
+
+    reader.close();
+}
+
+// 3. destory JVM
+    (jvm)->DestroyJavaVM();
+```
+Find example code at main.cpp of CSDK module
+
+## API List
+```
+    /**
+     * create a CarbonReaderBuilder object for building carbonReader,
+     * CarbonReaderBuilder object  can configure different parameter
+     *
+     * @param env JNIEnv
+     * @param path data store path
+     * @param tableName table name
+     * @return CarbonReaderBuilder object
+     */
+    jobject builder(JNIEnv *env, char *path, char *tableName);
+
+    /**
+     * Configure the projection column names of carbon reader
+     *
+     * @param argc argument counter
+     * @param argv argument vector
+     * @return CarbonReaderBuilder object
+     */
+    jobject projection(int argc, char *argv[]);
+
+    /**
+     *  build carbon reader with argument vector
+     *  it support multiple parameter
+     *  like: key=value
+     *  for example: fs.s3a.access.key=XXXX, XXXX is user's access key value
+     *
+     * @param argc argument counter
+     * @param argv argument vector
+     * @return CarbonReaderBuilder object
+     **/
+    jobject withHadoopConf(int argc, char *argv[]);
+
+    /**
+     * build carbonReader object for reading data
+     * it support read data from load disk
+     *
+     * @return carbonReader object
+     */
+    jobject build();
+
+    /**
+     * Whether it has next row data
+     *
+     * @return boolean value, if it has next row, return true. if it hasn't next row, return
false.
+     */
+    jboolean hasNext();
+
+    /**
+     * read next row from data
+     *
+     * @return object array of one row
+     */
+    jobjectArray readNextRow();
+
+    /**
+     * close the carbon reader
+     *
+     * @return  boolean value
+     */
+    jboolean close();
+
+```

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/examples/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index aa7b7c5..8dbab15 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -80,7 +80,7 @@
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
-      <version>4.2</version>
+      <version>${httpclient.version}</version>
     </dependency>
     <dependency>
       <groupId>org.scalatest</groupId>

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7b1d487..3dcce8d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -113,6 +113,7 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <snappy.version>1.1.2.6</snappy.version>
     <hadoop.version>2.7.2</hadoop.version>
+    <httpclient.version>4.2.5</httpclient.version>
     <scala.binary.version>2.11</scala.binary.version>
     <scala.version>2.11.8</scala.version>
     <hadoop.deps.scope>compile</hadoop.deps.scope>
@@ -483,6 +484,7 @@
       <id>hadoop-2.8</id>
       <properties>
         <hadoop.version>2.8.3</hadoop.version>
+        <httpclient.version>4.5.2</httpclient.version>
       </properties>
     </profile>
     <profile>

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/CSDK/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/store/CSDK/CMakeLists.txt b/store/CSDK/CMakeLists.txt
new file mode 100644
index 0000000..c3c57a0
--- /dev/null
+++ b/store/CSDK/CMakeLists.txt
@@ -0,0 +1,17 @@
+cmake_minimum_required (VERSION 2.8)
+project (CJDK)
+set(CMAKE_BUILD_TYPE Debug)
+SET (CMAKE_INSTALL_RPATH_USE_LINK_PATH true)
+
+find_package (JNI REQUIRED)
+
+include_directories (${JNI_INCLUDE_DIRS})
+
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+set (SOURCE_FILES CarbonReader.cpp CarbonReader.h main.cpp)
+
+add_executable (CJDK ${SOURCE_FILES})
+get_filename_component (JAVA_JVM_LIBRARY_DIR ${JAVA_JVM_LIBRARY} DIRECTORY)
+message (${JAVA_JVM_LIBRARY_DIR})
+target_link_libraries (CJDK ${JAVA_JVM_LIBRARY} )
+

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/CSDK/CarbonReader.cpp
----------------------------------------------------------------------
diff --git a/store/CSDK/CarbonReader.cpp b/store/CSDK/CarbonReader.cpp
new file mode 100644
index 0000000..0400957
--- /dev/null
+++ b/store/CSDK/CarbonReader.cpp
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "CarbonReader.h"
+#include <jni.h>
+
+jobject CarbonReader::builder(JNIEnv *env, char *path, char *tableName) {
+
+    jniEnv = env;
+    jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader");
+    jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder",
+        "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+    jstring jpath = env->NewStringUTF(path);
+    jstring jtableName = env->NewStringUTF(tableName);
+    jvalue args[2];
+    args[0].l = jpath;
+    args[1].l = jtableName;
+    carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID,
args);
+    return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::builder(JNIEnv *env, char *path) {
+    jniEnv = env;
+    jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader");
+    jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder",
+        "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+    jstring jpath = env->NewStringUTF(path);
+    jvalue args[1];
+    args[0].l = jpath;
+    carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID,
args);
+    return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::projection(int argc, char *argv[]) {
+    jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+    jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "projection",
+        "([Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+    jclass objectArrayClass = jniEnv->FindClass("Ljava/lang/String;");
+    jobjectArray array = jniEnv->NewObjectArray(argc, objectArrayClass, NULL);
+    for (int i = 0; i < argc; ++i) {
+        jstring value = jniEnv->NewStringUTF(argv[i]);
+        jniEnv->SetObjectArrayElement(array, i, value);
+    }
+
+    jvalue args[1];
+    args[0].l = array;
+    carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID,
args);
+    return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::withHadoopConf(char *key, char *value) {
+    jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+    jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "withHadoopConf",
+        "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+
+    jvalue args[2];
+    args[0].l = jniEnv->NewStringUTF(key);
+    args[1].l = jniEnv->NewStringUTF(value);
+    carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID,
args);
+    return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::build() {
+    jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+    jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build",
+        "()Lorg/apache/carbondata/sdk/file/CarbonReader;");
+    carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID);
+    return carbonReaderObject;
+}
+
+jboolean CarbonReader::hasNext() {
+    jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+    jmethodID hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z");
+    unsigned char hasNext = jniEnv->CallBooleanMethod(carbonReaderObject, hasNextID);
+    return hasNext;
+}
+
+jobjectArray CarbonReader::readNextRow() {
+    jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+    jmethodID readNextRow2ID = jniEnv->GetMethodID(carbonReader, "readNextStringRow",
"()[Ljava/lang/Object;");
+    jobjectArray row = (jobjectArray) jniEnv->CallObjectMethod(carbonReaderObject, readNextRow2ID);
+    return row;
+}
+
+jboolean CarbonReader::close() {
+    jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+    jmethodID closeID = jniEnv->GetMethodID(carbonReader, "close", "()V");
+    jniEnv->CallBooleanMethod(carbonReaderObject, closeID);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/CSDK/CarbonReader.h
----------------------------------------------------------------------
diff --git a/store/CSDK/CarbonReader.h b/store/CSDK/CarbonReader.h
new file mode 100644
index 0000000..861f704
--- /dev/null
+++ b/store/CSDK/CarbonReader.h
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <jni.h>
+
+class CarbonReader {
+public:
+    /**
+     * jni env
+     */
+    JNIEnv *jniEnv;
+
+    /**
+     * carbonReaderBuilder object for building carbonReader
+     * it can configure some operation
+     */
+    jobject carbonReaderBuilderObject;
+
+    /**
+     * carbonReader object for reading data
+     */
+    jobject carbonReaderObject;
+
+    /**
+     * create a CarbonReaderBuilder object for building carbonReader,
+     * CarbonReaderBuilder object  can configure different parameter
+     *
+     * @param env JNIEnv
+     * @param path data store path
+     * @param tableName table name
+     * @return CarbonReaderBuilder object
+     */
+    jobject builder(JNIEnv *env, char *path, char *tableName);
+
+    /**
+     * create a CarbonReaderBuilder object for building carbonReader,
+     * CarbonReaderBuilder object  can configure different parameter
+     *
+     * @param env JNIEnv
+     * @param path data store path
+     * @return CarbonReaderBuilder object
+     * */
+    jobject builder(JNIEnv *env, char *path);
+
+    /**
+     * Configure the projection column names of carbon reader
+     *
+     * @param argc argument counter
+     * @param argv argument vector
+     * @return CarbonReaderBuilder object
+     */
+    jobject projection(int argc, char *argv[]);
+
+    /**
+     * configure parameter, including ak,sk and endpoint
+     *
+     * @param key key word
+     * @param value value
+     * @return CarbonReaderBuilder object
+     */
+    jobject withHadoopConf(char *key, char *value);
+
+    /**
+     * build carbonReader object for reading data
+     * it support read data from load disk
+     *
+     * @return carbonReader object
+     */
+    jobject build();
+
+    /**
+     * Whether it has next row data
+     *
+     * @return boolean value, if it has next row, return true. if it hasn't next row, return
false.
+     */
+    jboolean hasNext();
+
+    /**
+     * read next row from data
+     *
+     * @return object array of one row
+     */
+    jobjectArray readNextRow();
+
+    /**
+     * close the carbon reader
+     *
+     * @return  boolean value
+     */
+    jboolean close();
+};

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/CSDK/main.cpp
----------------------------------------------------------------------
diff --git a/store/CSDK/main.cpp b/store/CSDK/main.cpp
new file mode 100644
index 0000000..a8d1a87
--- /dev/null
+++ b/store/CSDK/main.cpp
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h>
+#include <jni.h>
+#include <stdlib.h>
+#include <iostream>
+#include <unistd.h>
+#include "CarbonReader.h"
+
+using namespace std;
+
+JavaVM *jvm;
+
+/**
+ * init jvm
+ *
+ * @return
+ */
+JNIEnv *initJVM() {
+    JNIEnv *env;
+    JavaVMInitArgs vm_args;
+    int parNum = 3;
+    int res;
+    JavaVMOption options[parNum];
+
+    options[0].optionString = "-Djava.compiler=NONE";
+    options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar";
+    options[2].optionString = "-verbose:jni";
+    vm_args.version = JNI_VERSION_1_8;
+    vm_args.nOptions = parNum;
+    vm_args.options = options;
+    vm_args.ignoreUnrecognized = JNI_FALSE;
+
+    res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args);
+    if (res < 0) {
+        fprintf(stderr, "\nCan't create Java VM\n");
+        exit(1);
+    }
+
+    return env;
+}
+
+/**
+ * test read data from local disk, without projection
+ *
+ * @param env  jni env
+ * @return
+ */
+bool readFromLocalWithoutProjection(JNIEnv *env) {
+
+    CarbonReader carbonReaderClass;
+    carbonReaderClass.builder(env, "../resources/carbondata");
+    carbonReaderClass.build();
+
+    printf("\nRead data from local  without projection:\n");
+
+    while (carbonReaderClass.hasNext()) {
+        jobjectArray row = carbonReaderClass.readNextRow();
+        jsize length = env->GetArrayLength(row);
+
+        int j = 0;
+        for (j = 0; j < length; j++) {
+            jobject element = env->GetObjectArrayElement(row, j);
+            char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+            printf("%s\t", str);
+        }
+        printf("\n");
+    }
+
+    carbonReaderClass.close();
+}
+
+/**
+ * test read data from local disk
+ *
+ * @param env  jni env
+ * @return
+ */
+bool readFromLocal(JNIEnv *env) {
+
+    CarbonReader reader;
+    reader.builder(env, "../resources/carbondata", "test");
+
+    char *argv[11];
+    argv[0] = "stringField";
+    argv[1] = "shortField";
+    argv[2] = "intField";
+    argv[3] = "longField";
+    argv[4] = "doubleField";
+    argv[5] = "boolField";
+    argv[6] = "dateField";
+    argv[7] = "timeField";
+    argv[8] = "decimalField";
+    argv[9] = "varcharField";
+    argv[10] = "arrayField";
+    reader.projection(11, argv);
+
+    reader.build();
+
+    printf("\nRead data from local:\n");
+
+    while (reader.hasNext()) {
+        jobjectArray row = reader.readNextRow();
+        jsize length = env->GetArrayLength(row);
+
+        int j = 0;
+        for (j = 0; j < length; j++) {
+            jobject element = env->GetObjectArrayElement(row, j);
+            char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+            printf("%s\t", str);
+        }
+        printf("\n");
+    }
+
+    reader.close();
+}
+
+/**
+ * read data from S3
+ * parameter is ak sk endpoint
+ *
+ * @param env jni env
+ * @param argv argument vector
+ * @return
+ */
+bool readFromS3(JNIEnv *env, char *argv[]) {
+    CarbonReader reader;
+
+    char *args[3];
+    // "your access key"
+    args[0] = argv[1];
+    // "your secret key"
+    args[1] = argv[2];
+    // "your endPoint"
+    args[2] = argv[3];
+
+    reader.builder(env, "s3a://sdk/WriterOutput", "test");
+    reader.withHadoopConf("fs.s3a.access.key", argv[1]);
+    reader.withHadoopConf("fs.s3a.secret.key", argv[2]);
+    reader.withHadoopConf("fs.s3a.endpoint", argv[3]);
+    reader.build();
+    printf("\nRead data from S3:\n");
+    while (reader.hasNext()) {
+        jobjectArray row = reader.readNextRow();
+        jsize length = env->GetArrayLength(row);
+
+        int j = 0;
+        for (j = 0; j < length; j++) {
+            jobject element = env->GetObjectArrayElement(row, j);
+            char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+            printf("%s\t", str);
+        }
+        printf("\n");
+    }
+
+    reader.close();
+}
+
+/**
+ * This a example for C++ interface to read carbon file
+ * If you want to test read data fromS3, please input the parameter: ak sk endpoint
+ *
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return
+ */
+int main(int argc, char *argv[]) {
+    // init jvm
+    JNIEnv *env;
+    env = initJVM();
+
+    if (argc > 3) {
+        readFromS3(env, argv);
+    } else {
+        readFromLocalWithoutProjection(env);
+        readFromLocal(env);
+    }
+    cout << "destory jvm\n\n";
+    (jvm)->DestroyJavaVM();
+
+    cout << "\nfinish destory jvm";
+    fprintf(stdout, "Java VM destory.\n");
+    return 0;
+}
+

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/sdk/pom.xml
----------------------------------------------------------------------
diff --git a/store/sdk/pom.xml b/store/sdk/pom.xml
index ea720a2..2424113 100644
--- a/store/sdk/pom.xml
+++ b/store/sdk/pom.xml
@@ -39,6 +39,11 @@
       <artifactId>hadoop-aws</artifactId>
       <version>${hadoop.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>${httpclient.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index be809e6..1f1794c 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -24,6 +24,7 @@ import java.util.List;
 
 import org.apache.carbondata.common.annotations.InterfaceAudience;
 import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.util.CarbonTaskInfo;
 import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
 
@@ -91,6 +92,35 @@ public class CarbonReader<T> {
   }
 
   /**
+   * Read and return next string row object
+   * limitation: only single dimension Array is supported
+   * TODO: support didfferent data type
+   */
+  public Object[] readNextStringRow() throws IOException, InterruptedException {
+    validateReader();
+    T t = currentReader.getCurrentValue();
+    Object[] objects = (Object[]) t;
+    String[] strings = new String[objects.length];
+    for (int i = 0; i < objects.length; i++) {
+      if (objects[i] instanceof Object[]) {
+        Object[] arrayString = (Object[]) objects[i];
+        StringBuffer stringBuffer = new StringBuffer();
+        stringBuffer.append(String.valueOf(arrayString[0]));
+        if (arrayString.length > 1) {
+          for (int j = 1; j < arrayString.length; j++) {
+            stringBuffer.append(CarbonCommonConstants.ARRAY_SEPARATOR)
+                .append(String.valueOf(arrayString[j]));
+          }
+        }
+        strings[i] = stringBuffer.toString();
+      } else {
+        strings[i] = String.valueOf(objects[i]);
+      }
+    }
+    return strings;
+  }
+
+  /**
    * Return a new {@link CarbonReaderBuilder} instance
    *
    * @param tablePath table store path

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
index af3480f..151d57c 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
@@ -102,6 +102,22 @@ public class CarbonReaderBuilder {
   }
 
   /**
+   * configure hadoop configuration with key value
+   *
+   * @param key   key word
+   * @param value value
+   * @return this object
+   */
+  public CarbonReaderBuilder withHadoopConf(String key, String value) {
+    if (this.hadoopConf == null) {
+      this.hadoopConf = new Configuration();
+
+    }
+    this.hadoopConf.set(key, value);
+    return this;
+  }
+
+  /**
    * Build CarbonReader
    *
    * @param <T>


Mime
View raw message