Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id AB619200CFC for ; Wed, 23 Aug 2017 22:55:03 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id A9BF4169AAF; Wed, 23 Aug 2017 20:55:03 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 0B041169A99 for ; Wed, 23 Aug 2017 22:55:01 +0200 (CEST) Received: (qmail 56844 invoked by uid 500); 23 Aug 2017 20:54:59 -0000 Mailing-List: contact commits-help@gora.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@gora.apache.org Delivered-To: mailing list commits@gora.apache.org Received: (qmail 56827 invoked by uid 99); 23 Aug 2017 20:54:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 23 Aug 2017 20:54:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 62FBCF5EE0; Wed, 23 Aug 2017 20:54:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: lewismc@apache.org To: commits@gora.apache.org Date: Wed, 23 Aug 2017 20:55:00 -0000 Message-Id: <6811b58396454da3adf253313da84c8a@git.apache.org> In-Reply-To: <82e06533fe704bfdb7827a835514d66c@git.apache.org> References: <82e06533fe704bfdb7827a835514d66c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [02/37] gora git commit: Adding Cassandra Mapping file reading archived-at: Wed, 23 Aug 2017 20:55:03 -0000 Adding Cassandra Mapping file reading Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/a7b99f83 Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/a7b99f83 Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/a7b99f83 Branch: refs/heads/master Commit: a7b99f83aaf385cbe6772a5a0c7843b65153ec57 Parents: cd3522a Author: madhawa-gunasekara Authored: Thu Jun 22 15:36:48 2017 +0530 Committer: madhawa Committed: Fri Jun 23 00:52:39 2017 +0530 ---------------------------------------------------------------------- gora-cassandra-cql/pom.xml | 47 +---- .../org/apache/gora/cassandra/bean/Field.java | 54 ++++++ .../apache/gora/cassandra/bean/KeySpace.java | 84 +++++++++ .../apache/gora/cassandra/bean/PrimaryKey.java | 7 + .../gora/cassandra/store/CassandraMapping.java | 60 +++++++ .../cassandra/store/CassandraQueryFactory.java | 58 ++++++ .../gora/cassandra/store/CassandraStore.java | 178 +++++++++++++++++-- .../apache/gora/cassandra/store/Constants.java | 75 ++++---- .../src/test/conf/gora-cassandra-mapping.xml | 78 ++++---- .../gora/cassandra/GoraCassandraTestDriver.java | 3 +- .../cassandra/store/TestCassandraStore.java | 9 +- 11 files changed, 514 insertions(+), 139 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/a7b99f83/gora-cassandra-cql/pom.xml ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/pom.xml b/gora-cassandra-cql/pom.xml index b17fd41..11da4fc 100644 --- a/gora-cassandra-cql/pom.xml +++ b/gora-cassandra-cql/pom.xml @@ -17,22 +17,6 @@ --> - 4.0.0 @@ -65,6 +49,7 @@ + 17.0 * org.apache.gora.cassandra*;version="${project.version}";-noimport:=true @@ -136,12 +121,12 @@ com.datastax.cassandra cassandra-driver-core ${cassandra-driver.version} - - - com.google.guava - guava - - + + + + com.datastax.cassandra + cassandra-driver-mapping + ${cassandra-driver.version} @@ -165,26 +150,11 @@ - - - - - - - - - - - - - com.google.guava guava + ${guava.version} @@ -192,7 +162,6 @@ jdom - org.apache.avro avro http://git-wip-us.apache.org/repos/asf/gora/blob/a7b99f83/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/Field.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/Field.java b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/Field.java new file mode 100644 index 0000000..2bd997f --- /dev/null +++ b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/Field.java @@ -0,0 +1,54 @@ +package org.apache.gora.cassandra.bean; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by madhawa on 6/22/17. + */ +public class Field { + + private String fieldName; + + private String columnName; + + private String type; + + public Field() { + properties = new HashMap<>(2); + } + + private Map properties; + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public void setColumnName(String columnName) { + this.columnName = columnName; + } + + public void addProperty(String key, String value) { + properties.put(key, value); + } + + public String getFieldName() { + return fieldName; + } + + public String getColumnName() { + return columnName; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getProperty(String key) { + return this.properties.get(key); + } +} http://git-wip-us.apache.org/repos/asf/gora/blob/a7b99f83/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/KeySpace.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/KeySpace.java b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/KeySpace.java new file mode 100644 index 0000000..af563da --- /dev/null +++ b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/KeySpace.java @@ -0,0 +1,84 @@ +package org.apache.gora.cassandra.bean; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This class represents the Cassandra Keyspace. + */ +public class KeySpace { + public String getName() { + return name; + } + + public boolean isDurableWritesEnabled() { + return durableWritesEnabled; + } + + public PlacementStrategy getPlacementStrategy() { + return placementStrategy; + } + + public int getReplicationFactor() { + return replicationFactor; + } + + public Map getDataCenters() { + return dataCenters; + } + + public void addDataCenter(String key, Integer value) { + this.dataCenters.put(key, value); + } + + private String name; + + private Map properties; + + private boolean durableWritesEnabled; + + public KeySpace() { + this.properties = new HashMap<>(); + } + + public enum PlacementStrategy { + SimpleStrategy, + NetworkTopologyStrategy, + } + + public void setPlacementStrategy(PlacementStrategy placementStrategy) { + this.placementStrategy = placementStrategy; + if(placementStrategy.equals(PlacementStrategy.NetworkTopologyStrategy) && this.dataCenters == null) { + this.dataCenters = new HashMap<>(); + } + } + + private PlacementStrategy placementStrategy; + + public void setReplicationFactor(int replicationFactor) { + this.replicationFactor = replicationFactor; + } + + private int replicationFactor; + + private Map dataCenters; + + private List tables; + + public void addProperty(String key, String value) { + this.properties.put(key, value); + } + + public String getProperty(String key) { + return this.properties.get(key); + } + + public void setName(String name) { + this.name = name; + } + + public void setDurableWritesEnabled(boolean durableWritesEnabled) { + this.durableWritesEnabled = durableWritesEnabled; + } +} http://git-wip-us.apache.org/repos/asf/gora/blob/a7b99f83/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/PrimaryKey.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/PrimaryKey.java b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/PrimaryKey.java new file mode 100644 index 0000000..d3477f2 --- /dev/null +++ b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/bean/PrimaryKey.java @@ -0,0 +1,7 @@ +package org.apache.gora.cassandra.bean; + +/** + * Created by madhawa on 6/22/17. + */ +public class PrimaryKey { +} http://git-wip-us.apache.org/repos/asf/gora/blob/a7b99f83/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraMapping.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraMapping.java b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraMapping.java new file mode 100644 index 0000000..e533d57 --- /dev/null +++ b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraMapping.java @@ -0,0 +1,60 @@ +package org.apache.gora.cassandra.store; + +import org.apache.gora.cassandra.bean.Field; +import org.apache.gora.cassandra.bean.KeySpace; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by madhawa on 6/22/17. + */ +public class CassandraMapping { + + public KeySpace getKeySpace() { + return keySpace; + } + + public void setKeySpace(KeySpace keySpace) { + this.keySpace = keySpace; + } + + private KeySpace keySpace; + + public List getFieldList() { + return fieldList; + } + + private List fieldList; + + private Map tableProperties; + + public CassandraMapping() { + this.fieldList = new ArrayList<>(); + this.tableProperties = new HashMap<>(); + } + + public void setCoreName(String coreName) { + this.coreName = coreName; + } + + public String getCoreName() { + return coreName; + } + + private String coreName; + + public void addCassandraField(Field field) { + this.fieldList.add(field); + } + + public void addProperty(String key, String value) { + this.tableProperties.put(key,value); + } + + public String getProperty(String key) { + return this.tableProperties.get(key); + } +} http://git-wip-us.apache.org/repos/asf/gora/blob/a7b99f83/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraQueryFactory.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraQueryFactory.java b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraQueryFactory.java new file mode 100644 index 0000000..e70eaee --- /dev/null +++ b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraQueryFactory.java @@ -0,0 +1,58 @@ +package org.apache.gora.cassandra.store; + +import org.apache.gora.cassandra.bean.Field; +import org.apache.gora.cassandra.bean.KeySpace; + +import java.util.Map; + +class CassandraQueryFactory { + + static String getCreateKeySpaceQuery(CassandraMapping mapping) { + KeySpace keySpace = mapping.getKeySpace(); + StringBuilder stringBuffer = new StringBuilder(); + stringBuffer.append("CREATE KEYSPACE ").append(keySpace.getName()).append(" WITH REPLICATION = { 'class' : "); + KeySpace.PlacementStrategy placementStrategy = keySpace.getPlacementStrategy(); + stringBuffer.append("'").append(placementStrategy).append("'").append(", ").append("'"); + switch (placementStrategy) { + case SimpleStrategy: + stringBuffer.append("replication_factor").append("'").append(" : ").append(keySpace.getReplicationFactor()).append(" }"); + break; + case NetworkTopologyStrategy: + boolean isCommaNeeded = false; + for (Map.Entry entry : keySpace.getDataCenters().entrySet()) { + if (isCommaNeeded) { + stringBuffer.append(", '"); + } + stringBuffer.append(entry.getKey()).append("'").append(" : ").append(entry.getValue()); + isCommaNeeded = true; + } + stringBuffer.append(" }"); + break; + } + + if(keySpace.isDurableWritesEnabled()) { + stringBuffer.append(" AND DURABLE_WRITES = ").append(keySpace.isDurableWritesEnabled()); + } + return stringBuffer.toString(); + } + + static String getCreateTableQuery(CassandraMapping mapping) { + StringBuilder stringBuffer = new StringBuilder(); +stringBuffer.append("CREATE TABLE ").append(mapping.getCoreName()).append(" ("); + boolean isCommaNeeded = false; + for(Field field : mapping.getFieldList()) { + if(isCommaNeeded) { + stringBuffer.append(", "); + } + stringBuffer.append(field.getColumnName()).append(" ").append(field.getType()); + boolean isStaticColumn = Boolean.parseBoolean(field.getProperty("static")); + if( isStaticColumn) { + stringBuffer.append(" STATIC"); + } + isCommaNeeded =true; + } + + return stringBuffer.toString(); + } + +} http://git-wip-us.apache.org/repos/asf/gora/blob/a7b99f83/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java index b4c6e68..4bd1ad3 100644 --- a/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java +++ b/gora-cassandra-cql/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java @@ -17,15 +17,24 @@ package org.apache.gora.cassandra.store; +import com.datastax.driver.mapping.Mapper; +import com.datastax.driver.mapping.MappingManager; import com.datastax.driver.core.*; import com.datastax.driver.core.policies.*; +import org.apache.gora.cassandra.bean.Field; +import org.apache.gora.cassandra.bean.KeySpace; import org.apache.gora.persistency.BeanFactory; import org.apache.gora.persistency.impl.PersistentBase; import org.apache.gora.query.PartitionQuery; import org.apache.gora.query.Query; import org.apache.gora.query.Result; +import org.apache.gora.store.DataStoreFactory; import org.apache.gora.store.impl.DataStoreBase; import org.apache.hadoop.conf.Configuration; +import org.jdom.Attribute; +import org.jdom.Document; +import org.jdom.Element; +import org.jdom.input.SAXBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,25 +46,18 @@ import java.util.Properties; public class CassandraStore extends DataStoreBase { - /** - * Consistency property level for Cassandra column families - */ - private static final String COL_FAM_CL = "cf.consistency.level"; - - /** - * Consistency property level for Cassandra read operations. - */ - private static final String READ_OP_CL = "read.consistency.level"; - - /** - * Consistency property level for Cassandra write operations. - */ - private static final String WRITE_OP_CL = "write.consistency.level"; + private static final String DEFAULT_MAPPING_FILE = "gora-cassandra-mapping.xml"; public static final Logger LOG = LoggerFactory.getLogger(CassandraStore.class); private Cluster cluster; + private CassandraMapping mapping; + + private boolean isUseCassandraMappingManager; + + private Mapper mapper; + private Session session; public CassandraStore() { @@ -67,10 +69,19 @@ public class CassandraStore extends DataStoreBase extends DataStoreBase keyspaces = doc.getRootElement().getChildren("keyspace"); + + List classes = doc.getRootElement().getChildren("class"); + + boolean classMatched = false; + for (Element classElement : classes) { + if (classElement.getAttributeValue("keyClass").equals( + keyClass.getCanonicalName()) + && classElement.getAttributeValue("name").equals( + persistentClass.getCanonicalName())) { + + classMatched = true; + String tableName = getSchemaName( + classElement.getAttributeValue("table"), persistentClass); + map.setCoreName(tableName); + + List classAttributes = classElement.getAttributes(); + for (Object anAttributeList : classAttributes) { + Attribute attribute = (Attribute) anAttributeList; + String attributeName = attribute.getName(); + String attributeValue = attribute.getValue(); + map.addProperty(attributeName, attributeValue); + } + + List fields = classElement.getChildren("field"); + + for (Element field : fields) { + Field cassandraField = new Field(); + + List fieldAttributes = field.getAttributes(); + for (Object anAttributeList : fieldAttributes) { + Attribute attribute = (Attribute) anAttributeList; + String attributeName = attribute.getName(); + String attributeValue = attribute.getValue(); + switch (attributeName) { + case "name": + cassandraField.setFieldName(attributeValue); + break; + case "column": + cassandraField.setColumnName(attributeValue); + break; + default: + cassandraField.addProperty(attributeName, attributeValue); + break; + } + } + map.addCassandraField(cassandraField); + } + break; + } + LOG.warn("Check that 'keyClass' and 'name' parameters in gora-solr-mapping.xml " + + "match with intended values. A mapping mismatch has been found therefore " + + "no mapping has been initialized for class mapping at position " + + " {} in mapping file.", classes.indexOf(classElement)); + } + if (!classMatched) { + LOG.error("Check that 'keyClass' and 'name' parameters in {} no mapping has been initialized for {} class mapping", filename, persistentClass); + } + + String keyspaceName = map.getProperty("keyspace"); + if (keyspaceName != null) { + KeySpace keyspace = null; + for (Element keyspaceElement : keyspaces) { + if (keyspaceName.equals(keyspaceElement.getAttributeValue("name"))) { + keyspace = new KeySpace(); + List fieldAttributes = keyspaceElement.getAttributes(); + for (Object attributeObject : fieldAttributes) { + Attribute attribute = (Attribute) attributeObject; + String attributeName = attribute.getName(); + String attributeValue = attribute.getValue(); + switch (attributeName) { + case "name": + keyspace.setName(attributeValue); + break; + case "durableWrite": + keyspace.setDurableWritesEnabled(Boolean.parseBoolean(attributeValue)); + break; + default: + keyspace.addProperty(attributeName, attributeValue); + break; + } + } + Element placementStrategy = keyspaceElement.getChild("placementStrategy"); + switch (KeySpace.PlacementStrategy.valueOf(placementStrategy.getAttributeValue("name"))) { + case SimpleStrategy: + keyspace.setPlacementStrategy(KeySpace.PlacementStrategy.SimpleStrategy); + keyspace.setReplicationFactor(Integer.parseInt(placementStrategy.getAttributeValue("replication_factor"))); + break; + case NetworkTopologyStrategy: + List dataCenters = placementStrategy.getChildren("datacenter"); + keyspace.setPlacementStrategy(KeySpace.PlacementStrategy.NetworkTopologyStrategy); + for(Element dataCenter : dataCenters) { + String dataCenterName = dataCenter.getAttributeValue("name"); + Integer dataCenterReplicationFactor = Integer.valueOf(dataCenter.getAttributeValue("replication_factor")); + keyspace.addDataCenter(dataCenterName, dataCenterReplicationFactor); + } + break; + } + map.setKeySpace(keyspace); + break; + } + + } + + } + + } catch (Exception ex) { + throw new IOException(ex); + } + + return map; + } + private void populateSettings(Cluster.Builder builder, Properties properties) { String serversParam = properties.getProperty(Constants.CASSANDRA_SERVERS); String[] servers = serversParam.split(","); @@ -416,7 +547,11 @@ public class CassandraStore extends DataStoreBase extends DataStoreBase - - - - - - - - - + + + + - - - - + + + + + + - - - - - - - - + + + + + + + - - - - - - - - - - + + + + + + + - - - + + + + + + + + + + + + + - + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/gora/blob/a7b99f83/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/GoraCassandraTestDriver.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/GoraCassandraTestDriver.java b/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/GoraCassandraTestDriver.java index b36aca4..ccb9b97 100644 --- a/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/GoraCassandraTestDriver.java +++ b/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/GoraCassandraTestDriver.java @@ -40,8 +40,7 @@ import org.slf4j.LoggerFactory; * This driver is the base for all test cases that require an embedded Cassandra * server. In this case we draw on Hector's @see EmbeddedServerHelper. * It starts (setUp) and stops (tearDown) embedded Cassandra server. - * - * @author lewismc + * */ public class GoraCassandraTestDriver extends GoraTestDriver { http://git-wip-us.apache.org/repos/asf/gora/blob/a7b99f83/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/store/TestCassandraStore.java ---------------------------------------------------------------------- diff --git a/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/store/TestCassandraStore.java b/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/store/TestCassandraStore.java index 9518891..44c54b9 100644 --- a/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/store/TestCassandraStore.java +++ b/gora-cassandra-cql/src/test/java/org/apache/gora/cassandra/store/TestCassandraStore.java @@ -23,18 +23,14 @@ */ package org.apache.gora.cassandra.store; -import java.io.IOException; - import org.apache.gora.cassandra.GoraCassandraTestDriver; -import org.apache.gora.examples.generated.Employee; -import org.apache.gora.examples.generated.WebPage; -import org.apache.gora.store.DataStore; -import org.apache.gora.store.DataStoreFactory; import org.apache.gora.store.DataStoreTestBase; import org.apache.hadoop.conf.Configuration; import org.junit.Before; import org.junit.Ignore; +import java.io.IOException; + /** * Test for CassandraStore. */ @@ -42,6 +38,7 @@ public class TestCassandraStore extends DataStoreTestBase{ private Configuration conf; + static { setTestDriver(new GoraCassandraTestDriver()); }