carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] carbondata git commit: Added data map interfaces
Date Thu, 15 Jun 2017 17:29:21 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master c3cde5af1 -> 7bb822031


Added data map interfaces

Fixed comments


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b4343469
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b4343469
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b4343469

Branch: refs/heads/master
Commit: b434346922632899b2bf77d5e4d5469caf63d40a
Parents: c3cde5a
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Sun Jun 4 18:32:48 2017 +0530
Committer: jackylk <jacky.likun@huawei.com>
Committed: Fri Jun 16 01:18:40 2017 +0800

----------------------------------------------------------------------
 .../carbondata/core/events/ChangeEvent.java     |  35 +++++++
 .../carbondata/core/events/EventListener.java   |  25 +++++
 .../carbondata/core/indexstore/Blocklet.java    |  42 ++++++++
 .../carbondata/core/indexstore/DataMap.java     |  54 ++++++++++
 .../core/indexstore/DataMapDistributable.java   |  56 ++++++++++
 .../core/indexstore/DataMapStoreManager.java    | 105 +++++++++++++++++++
 .../carbondata/core/indexstore/DataMapType.java |  34 ++++++
 .../core/indexstore/DataMapWriter.java          |  50 +++++++++
 .../core/indexstore/TableDataMap.java           |  94 +++++++++++++++++
 9 files changed, 495 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/events/ChangeEvent.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/events/ChangeEvent.java b/core/src/main/java/org/apache/carbondata/core/events/ChangeEvent.java
new file mode 100644
index 0000000..de02c64
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/events/ChangeEvent.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.events;
+
+/**
+ * Change event for any updates in store.
+ */
+public interface ChangeEvent<T> {
+
+  EventType getEventType();
+
+  T getEventData();
+
+  void setEventData(T data);
+
+  enum EventType {
+    INSERT,DELETE,UPDATE,REFRESH
+  }
+}
+
+

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/events/EventListener.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/events/EventListener.java b/core/src/main/java/org/apache/carbondata/core/events/EventListener.java
new file mode 100644
index 0000000..adf45b2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/events/EventListener.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.events;
+
+/**
+ * Event listener
+ */
+public interface EventListener {
+
+  void fireEvent(ChangeEvent event);
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
new file mode 100644
index 0000000..597c46c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.io.Serializable;
+
+/**
+ * Blocklet
+ */
+public class Blocklet implements Serializable {
+
+  private String path;
+
+  private String blockletId;
+
+  public Blocklet(String path, String blockletId) {
+    this.path = path;
+    this.blockletId = blockletId;
+  }
+
+  public String getPath() {
+    return path;
+  }
+
+  public String getBlockletId() {
+    return blockletId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java
new file mode 100644
index 0000000..2651f15
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.util.List;
+
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+/**
+ * Interface for adding and retrieving index data.
+ */
+public interface DataMap {
+
+  /**
+   * Give the writer to write the data.
+   *
+   * @return
+   */
+  DataMapWriter getWriter();
+
+  /**
+   * It is called to load the data map to memory or to initialize it.
+   */
+  void init(String path);
+
+  /**
+   * Prune the datamap with filter expression. It returns the list of
+   * blocklets where these filters can exist.
+   *
+   * @param filterExp
+   * @return
+   */
+  List<Blocklet> prune(FilterResolverIntf filterExp);
+
+  /**
+   * Clear complete index table and release memory.
+   */
+  void clear();
+
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapDistributable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapDistributable.java
b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapDistributable.java
new file mode 100644
index 0000000..4c379f3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapDistributable.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import org.apache.carbondata.core.datastore.block.Distributable;
+
+/**
+ * Distributable class for datamap.
+ */
+public abstract class DataMapDistributable implements Distributable {
+
+  private String tablePath;
+
+  private String segmentId;
+
+  private String dataMapName;
+
+  public String getTablePath() {
+    return tablePath;
+  }
+
+  public void setTablePath(String tablePath) {
+    this.tablePath = tablePath;
+  }
+
+  public String getSegmentId() {
+    return segmentId;
+  }
+
+  public void setSegmentId(String segmentId) {
+    this.segmentId = segmentId;
+  }
+
+  public String getDataMapName() {
+    return dataMapName;
+  }
+
+  public void setDataMapName(String dataMapName) {
+    this.dataMapName = dataMapName;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
new file mode 100644
index 0000000..06638ad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+
+/**
+ * It maintains all the index tables in it.
+ */
+public class DataMapStoreManager {
+
+  private static DataMapStoreManager instance = new DataMapStoreManager();
+
+  private Map<DataMapType, Map<String, TableDataMap>> dataMapMappping = new HashMap<>();
+
+  private static final LogService LOGGER =
+      LogServiceFactory.getLogService(DataMapStoreManager.class.getName());
+
+  private DataMapStoreManager() {
+
+  }
+
+  /**
+   * Get the datamap for reading data.
+   *
+   * @param dataMapName
+   * @param mapType
+   * @return
+   */
+  public TableDataMap getDataMap(AbsoluteTableIdentifier identifier, String dataMapName,
+      DataMapType mapType) {
+    Map<String, TableDataMap> map = dataMapMappping.get(mapType);
+    TableDataMap dataMap = null;
+    if (map == null) {
+      throw new RuntimeException("Datamap does not exist");
+    } else {
+      dataMap = map.get(dataMapName);
+      if (dataMap == null) {
+        throw new RuntimeException("Datamap does not exist");
+      }
+    }
+    // Initialize datamap
+    dataMap.init(identifier, dataMapName);
+    return dataMap;
+  }
+
+  /**
+   * Create new datamap instance using datamap type and path
+   *
+   * @param mapType
+   * @return
+   */
+  public TableDataMap createTableDataMap(AbsoluteTableIdentifier identifier, DataMapType
mapType,
+      String dataMapName) {
+    Map<String, TableDataMap> map = dataMapMappping.get(mapType);
+    if (map == null) {
+      map = new HashMap<>();
+      dataMapMappping.put(mapType, map);
+    }
+    TableDataMap dataMap = map.get(dataMapName);
+    if (dataMap != null) {
+      throw new RuntimeException("Already datamap exists in that path with type " + mapType);
+    }
+
+    try {
+      //TODO create datamap using @mapType.getClassName())
+    } catch (Exception e) {
+      LOGGER.error(e);
+    }
+    dataMap.init(identifier, dataMapName);
+    map.put(dataMapName, dataMap);
+    return dataMap;
+  }
+
+  public void clearDataMap(String dataMapName, DataMapType mapType) {
+    Map<String, TableDataMap> map = dataMapMappping.get(mapType);
+    if (map != null && map.get(dataMapName) != null) {
+      map.remove(dataMapName).clear();
+    }
+  }
+
+  public static DataMapStoreManager getInstance() {
+    return instance;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java
new file mode 100644
index 0000000..b6a0f5b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+/**
+ * Datamap type
+ */
+public enum DataMapType {
+  BLOCKLET("org.apache.carbondata.datamap.BlockletDataMap");
+
+  private String className;
+
+  DataMapType(String className) {
+    this.className = className;
+  }
+
+  public String getClassName() {
+    return className;
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapWriter.java
new file mode 100644
index 0000000..bd8be09
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapWriter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.io.DataOutput;
+
+/**
+ * Data Map writer
+ */
+public interface DataMapWriter<T> {
+
+  /**
+   * Initialize the data map writer with output stream
+   *
+   * @param outStream
+   */
+  void init(DataOutput outStream);
+
+  /**
+   * Add the index row to the in-memory store.
+   */
+  void writeData(T data);
+
+  /**
+   * Get the added row count
+   *
+   * @return
+   */
+  int getRowCount();
+
+  /**
+   * Finish writing of data map table, otherwise it will not be allowed to read.
+   */
+  void finish();
+
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java
new file mode 100644
index 0000000..e1532c8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.util.List;
+
+import org.apache.carbondata.core.events.EventListener;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+/**
+ * DataMap at the table level, user can add any number of datamaps for one table. Depends
+ * on the filter condition it can prune the blocklets.
+ */
+public interface TableDataMap extends EventListener {
+
+  /**
+   * It is called to initialize and load the required table datamap metadata.
+   */
+  void init(AbsoluteTableIdentifier identifier, String dataMapName);
+
+  /**
+   * Gives the writer to write the metadata information of this datamap at table level.
+   *
+   * @return
+   */
+  DataMapWriter getWriter();
+
+  /**
+   * Create the datamap using the segmentid  and name.
+   *
+   * @param identifier
+   * @param segmentId
+   * @return
+   */
+  DataMap createDataMap(AbsoluteTableIdentifier identifier, String segmentId);
+
+  /**
+   * Pass the valid segments and prune the datamap using filter expression
+   *
+   * @param segmentIds
+   * @param filterExp
+   * @return
+   */
+  List<Blocklet> prune(List<String> segmentIds, FilterResolverIntf filterExp);
+
+  /**
+   * This is used for making the datamap distributable.
+   * It takes the valid segments and returns all the datamaps as distributable objects so
that
+   * it can be distributed across machines.
+   *
+   * @return
+   */
+  List<DataMapDistributable> toDistributable(List<String> segmentIds);
+
+  /**
+   * This method is used from any machine after it is distributed. It takes the distributable
object
+   * to prune the filters.
+   *
+   * @param distributable
+   * @param filterExp
+   * @return
+   */
+  List<Blocklet> prune(DataMapDistributable distributable, FilterResolverIntf filterExp);
+
+  /**
+   * This method checks whether the columns and the type of filters supported
+   * for this datamap or not
+   *
+   * @param filterExp
+   * @return
+   */
+  boolean isFiltersSupported(FilterResolverIntf filterExp);
+
+  /**
+   * Clears table level datamap
+   */
+  void clear();
+
+}


Mime
View raw message