Return-Path: X-Original-To: apmail-atlas-commits-archive@minotaur.apache.org Delivered-To: apmail-atlas-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id AD96418F96 for ; Tue, 16 Jun 2015 23:05:30 +0000 (UTC) Received: (qmail 32940 invoked by uid 500); 16 Jun 2015 23:05:30 -0000 Delivered-To: apmail-atlas-commits-archive@atlas.apache.org Received: (qmail 32918 invoked by uid 500); 16 Jun 2015 23:05:30 -0000 Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.incubator.apache.org Delivered-To: mailing list commits@atlas.incubator.apache.org Received: (qmail 32909 invoked by uid 99); 16 Jun 2015 23:05:30 -0000 Received: from Unknown (HELO spamd4-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Jun 2015 23:05:30 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd4-us-west.apache.org (ASF Mail Server at spamd4-us-west.apache.org) with ESMTP id DB699C0953 for ; Tue, 16 Jun 2015 23:05:29 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd4-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.77 X-Spam-Level: * X-Spam-Status: No, score=1.77 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, T_RP_MATCHES_RCVD=-0.01] autolearn=disabled Received: from mx1-us-east.apache.org ([10.40.0.8]) by localhost (spamd4-us-west.apache.org [10.40.0.11]) (amavisd-new, port 10024) with ESMTP id hJAbZZ89JIGP for ; Tue, 16 Jun 2015 23:05:12 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-east.apache.org (ASF Mail Server at mx1-us-east.apache.org) with SMTP id 616A64E5F2 for ; Tue, 16 Jun 2015 23:04:37 +0000 (UTC) Received: (qmail 30378 invoked by uid 99); 16 Jun 2015 23:04:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Jun 2015 23:04:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9C3ECE3C48; Tue, 16 Jun 2015 23:04:36 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: venkatesh@apache.org To: commits@atlas.incubator.apache.org Date: Tue, 16 Jun 2015 23:05:05 -0000 Message-Id: In-Reply-To: <29f8990c10f64784b72a9cafbbc9676e@git.apache.org> References: <29f8990c10f64784b72a9cafbbc9676e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [31/58] [abbrv] incubator-atlas git commit: Refactor packages and scripts to Atlas (cherry picked from commit 414beba) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java deleted file mode 100755 index ac9e2fa..0000000 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.repository.graph; - -import com.thinkaurelius.titan.core.TitanGraph; -import com.thinkaurelius.titan.core.TitanVertex; -import com.tinkerpop.blueprints.Direction; -import com.tinkerpop.blueprints.Edge; -import com.tinkerpop.blueprints.Graph; -import com.tinkerpop.blueprints.GraphQuery; -import com.tinkerpop.blueprints.Vertex; -import org.apache.hadoop.metadata.repository.Constants; -import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance; -import org.apache.hadoop.metadata.typesystem.persistence.Id; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Iterator; -import java.util.Set; -import java.util.UUID; - -/** - * Utility class for graph operations. - */ -public final class GraphHelper { - - private static final Logger LOG = LoggerFactory.getLogger(GraphHelper.class); - - private GraphHelper() { - } - - public static Vertex createVertexWithIdentity(Graph graph, - ITypedReferenceableInstance typedInstance, - Set superTypeNames) { - final Vertex vertexWithIdentity = createVertexWithoutIdentity( - graph, typedInstance.getTypeName(), typedInstance.getId(), superTypeNames); - - // add identity - final String guid = UUID.randomUUID().toString(); - vertexWithIdentity.setProperty(Constants.GUID_PROPERTY_KEY, guid); - - return vertexWithIdentity; - } - - public static Vertex createVertexWithoutIdentity(Graph graph, - String typeName, - Id typedInstanceId, - Set superTypeNames) { - final Vertex vertexWithoutIdentity = graph.addVertex(null); - - // add type information - vertexWithoutIdentity.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName); - - // add super types - for (String superTypeName : superTypeNames) { - ((TitanVertex) vertexWithoutIdentity).addProperty( - Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName); - } - - // add version information - vertexWithoutIdentity.setProperty(Constants.VERSION_PROPERTY_KEY, typedInstanceId.version); - - // add timestamp information - vertexWithoutIdentity.setProperty( - Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis()); - - return vertexWithoutIdentity; - } - - public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex, - String edgeLabel) { - LOG.debug("Adding edge for {} -> label {} -> {}", - fromVertex, edgeLabel, toVertex); - - return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel); - } - - public static Vertex findVertexByGUID(TitanGraph titanGraph, - String value) { - LOG.debug("Finding vertex for key={}, value={}", Constants.GUID_PROPERTY_KEY, value); - - GraphQuery query = titanGraph.query() - .has(Constants.GUID_PROPERTY_KEY, value); - Iterator results = query.vertices().iterator(); - // returning one since guid should be unique - return results.hasNext() ? results.next() : null; - } - - public static String vertexString(final Vertex vertex) { - StringBuilder properties = new StringBuilder(); - for (String propertyKey : vertex.getPropertyKeys()) { - properties.append(propertyKey) - .append("=").append(vertex.getProperty(propertyKey)) - .append(", "); - } - - return "v[" + vertex.getId() + "], Properties[" + properties + "]"; - } - - public static String edgeString(final Edge edge) { - return "e[" + edge.getLabel() + "], [" - + edge.getVertex(Direction.OUT) - + " -> " + edge.getLabel() + " -> " - + edge.getVertex(Direction.IN) - + "]"; - } - -/* - public static void dumpToLog(final Graph graph) { - LOG.debug("*******************Graph Dump****************************"); - LOG.debug("Vertices of {}", graph); - for (Vertex vertex : graph.getVertices()) { - LOG.debug(vertexString(vertex)); - } - - LOG.debug("Edges of {}", graph); - for (Edge edge : graph.getEdges()) { - LOG.debug(edgeString(edge)); - } - LOG.debug("*******************Graph Dump****************************"); - } -*/ -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphProvider.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphProvider.java deleted file mode 100755 index c2879c4..0000000 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphProvider.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.repository.graph; - -import com.google.inject.throwingproviders.CheckedProvider; -import com.tinkerpop.blueprints.Graph; - -public interface GraphProvider extends CheckedProvider { - - @Override - T get(); -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java deleted file mode 100755 index af81edc..0000000 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/TitanGraphProvider.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.repository.graph; - -import com.google.inject.Provides; -import com.thinkaurelius.titan.core.TitanFactory; -import com.thinkaurelius.titan.core.TitanGraph; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.hadoop.metadata.MetadataException; -import org.apache.hadoop.metadata.PropertiesUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Singleton; -import java.util.Iterator; - -/** - * Default implementation for Graph Provider that doles out Titan Graph. - */ -public class TitanGraphProvider implements GraphProvider { - - private static final Logger LOG = LoggerFactory.getLogger(TitanGraphProvider.class); - - /** - * Constant for the configuration property that indicates the prefix. - */ - private static final String METADATA_PREFIX = "metadata.graph."; - - private static Configuration getConfiguration() throws MetadataException { - PropertiesConfiguration configProperties = PropertiesUtil.getApplicationProperties(); - - Configuration graphConfig = new PropertiesConfiguration(); - - final Iterator iterator = configProperties.getKeys(); - while (iterator.hasNext()) { - String key = iterator.next(); - if (key.startsWith(METADATA_PREFIX)) { - String value = (String) configProperties.getProperty(key); - key = key.substring(METADATA_PREFIX.length()); - graphConfig.setProperty(key, value); - LOG.info("Using graph property {}={}", key, value); - } - } - - return graphConfig; - } - - @Override - @Singleton - @Provides - public TitanGraph get() { - Configuration config; - try { - config = getConfiguration(); - } catch (MetadataException e) { - throw new RuntimeException(e); - } - - return TitanFactory.open(config); - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/AttributeStores.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/AttributeStores.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/AttributeStores.java deleted file mode 100755 index 5b6ec34..0000000 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/AttributeStores.java +++ /dev/null @@ -1,655 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.repository.memory; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import it.unimi.dsi.fastutil.booleans.BooleanArrayList; -import it.unimi.dsi.fastutil.bytes.ByteArrayList; -import it.unimi.dsi.fastutil.doubles.DoubleArrayList; -import it.unimi.dsi.fastutil.floats.FloatArrayList; -import it.unimi.dsi.fastutil.ints.IntArrayList; -import it.unimi.dsi.fastutil.longs.LongArrayList; -import it.unimi.dsi.fastutil.shorts.ShortArrayList; -import org.apache.hadoop.metadata.repository.RepositoryException; -import org.apache.hadoop.metadata.typesystem.persistence.Id; -import org.apache.hadoop.metadata.typesystem.persistence.StructInstance; -import org.apache.hadoop.metadata.typesystem.types.AttributeInfo; -import org.apache.hadoop.metadata.typesystem.types.DataTypes; -import org.apache.hadoop.metadata.typesystem.types.IConstructableType; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class AttributeStores { - - private static final Object NULL_VAL = new Object(); - - static IAttributeStore createStore(AttributeInfo i) throws RepositoryException { - switch (i.dataType().getTypeCategory()) { - case PRIMITIVE: - if (i.dataType() == DataTypes.BOOLEAN_TYPE) { - return new BooleanAttributeStore(i); - } else if (i.dataType() == DataTypes.BYTE_TYPE) { - return new ByteAttributeStore(i); - } else if (i.dataType() == DataTypes.SHORT_TYPE) { - new ShortAttributeStore(i); - } else if (i.dataType() == DataTypes.INT_TYPE) { - return new IntAttributeStore(i); - } else if (i.dataType() == DataTypes.LONG_TYPE) { - return new LongAttributeStore(i); - } else if (i.dataType() == DataTypes.FLOAT_TYPE) { - return new FloatAttributeStore(i); - } else if (i.dataType() == DataTypes.DOUBLE_TYPE) { - return new DoubleAttributeStore(i); - } else if (i.dataType() == DataTypes.BIGINTEGER_TYPE) { - return new BigIntStore(i); - } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) { - return new BigDecimalStore(i); - } else if (i.dataType() == DataTypes.DATE_TYPE) { - return new DateStore(i); - } else if (i.dataType() == DataTypes.STRING_TYPE) { - return new StringStore(i); - } else if (i.dataType() == DataTypes.STRING_TYPE) { - return new StringStore(i); - } else { - throw new RepositoryException( - String.format("Unknown datatype %s", i.dataType())); - } - case ENUM: - return new IntAttributeStore(i); - case ARRAY: - return new ImmutableListStore(i); - case MAP: - return new ImmutableMapStore(i); - case STRUCT: - return new StructStore(i); - case CLASS: - return new IdStore(i); - default: - throw new RepositoryException( - String.format("Unknown Category for datatype %s", i.dataType())); - } - } - - static abstract class AbstractAttributeStore implements IAttributeStore { - final BooleanArrayList nullList; - final Map> hiddenVals; - AttributeInfo attrInfo; - - AbstractAttributeStore(AttributeInfo attrInfo) { - this.attrInfo = attrInfo; - this.nullList = new BooleanArrayList(); - hiddenVals = new HashMap>(); - } - - final void setNull(int pos, boolean flag) { - nullList.set(pos, flag); - } - - final boolean getNull(int pos) { - return nullList.get(pos); - } - - void storeHiddenVals(int pos, IConstructableType type, StructInstance instance) - throws RepositoryException { - List attrNames = type.getNames(attrInfo); - Map m = hiddenVals.get(pos); - if (m == null) { - m = new HashMap(); - hiddenVals.put(pos, m); - } - for (int i = 2; i < attrNames.size(); i++) { - String attrName = attrNames.get(i); - int nullPos = instance.fieldMapping().fieldNullPos.get(attrName); - int colPos = instance.fieldMapping().fieldPos.get(attrName); - if (instance.nullFlags[nullPos]) { - m.put(attrName, NULL_VAL); - } else { - //m.put(attrName, instance.bools[colPos]); - store(instance, colPos, attrName, m); - } - } - } - - void loadHiddenVals(int pos, IConstructableType type, StructInstance instance) - throws RepositoryException { - List attrNames = type.getNames(attrInfo); - Map m = hiddenVals.get(pos); - for (int i = 2; i < attrNames.size(); i++) { - String attrName = attrNames.get(i); - int nullPos = instance.fieldMapping().fieldNullPos.get(attrName); - int colPos = instance.fieldMapping().fieldPos.get(attrName); - Object val = m == null ? NULL_VAL : m.get(attrName); - if (val == NULL_VAL) { - instance.nullFlags[nullPos] = true; - } else { - instance.nullFlags[nullPos] = false; - load(instance, colPos, val); - } - } - } - - @Override - public void store(int pos, IConstructableType type, StructInstance instance) - throws RepositoryException { - List attrNames = type.getNames(attrInfo); - String attrName = attrNames.get(0); - int nullPos = instance.fieldMapping().fieldNullPos.get(attrName); - int colPos = instance.fieldMapping().fieldPos.get(attrName); - nullList.set(pos, instance.nullFlags[nullPos]); - - if (pos == nullList.size()) { - nullList.add(instance.nullFlags[nullPos]); - } else { - nullList.set(pos, instance.nullFlags[nullPos]); - } - //list.set(pos, instance.bools[colPos]); - store(instance, colPos, pos); - - if (attrNames.size() > 1) { - storeHiddenVals(pos, type, instance); - } - } - - @Override - public void load(int pos, IConstructableType type, StructInstance instance) - throws RepositoryException { - List attrNames = type.getNames(attrInfo); - String attrName = attrNames.get(0); - int nullPos = instance.fieldMapping().fieldNullPos.get(attrName); - int colPos = instance.fieldMapping().fieldPos.get(attrName); - - if (nullList.get(pos)) { - instance.nullFlags[nullPos] = true; - } else { - instance.nullFlags[nullPos] = false; - load(instance, colPos, pos); - } - - if (attrNames.size() > 1) { - loadHiddenVals(pos, type, instance); - } - } - - /* - * store the value from colPos in instance into the list. - */ - protected abstract void store(StructInstance instance, int colPos, int pos) - throws RepositoryException; - - /* - * load the value from pos in list into colPos in instance. - */ - protected abstract void load(StructInstance instance, int colPos, int pos) - throws RepositoryException; - - /* - * store the value from colPos in map as attrName - */ - protected abstract void store(StructInstance instance, int colPos, String attrName, - Map m); - - /* - * load the val into colPos in instance. - */ - protected abstract void load(StructInstance instance, int colPos, Object val); - - } - - static abstract class PrimitiveAttributeStore extends AbstractAttributeStore - implements IAttributeStore { - - - public PrimitiveAttributeStore(AttributeInfo attrInfo) { - super(attrInfo); - } - - } - - static class BooleanAttributeStore extends PrimitiveAttributeStore { - - final BooleanArrayList list; - - BooleanAttributeStore(AttributeInfo attrInfo) { - super(attrInfo); - this.list = new BooleanArrayList(); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.bools[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.bools[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.bools[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.bools[colPos] = (Boolean) val; - } - - @Override - public void ensureCapacity(int pos) throws RepositoryException { - list.size(pos + 1); - nullList.size(pos + 1); - } - } - - static class ByteAttributeStore extends PrimitiveAttributeStore { - - final ByteArrayList list; - - ByteAttributeStore(AttributeInfo attrInfo) { - super(attrInfo); - this.list = new ByteArrayList(); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.bytes[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.bytes[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.bytes[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.bytes[colPos] = (Byte) val; - } - - @Override - public void ensureCapacity(int pos) throws RepositoryException { - list.size(pos + 1); - nullList.size(pos + 1); - } - } - - static class ShortAttributeStore extends PrimitiveAttributeStore { - - final ShortArrayList list; - - ShortAttributeStore(AttributeInfo attrInfo) { - super(attrInfo); - this.list = new ShortArrayList(); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.shorts[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.shorts[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.shorts[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.shorts[colPos] = (Short) val; - } - - @Override - public void ensureCapacity(int pos) throws RepositoryException { - list.size(pos + 1); - nullList.size(pos + 1); - } - } - - static class IntAttributeStore extends PrimitiveAttributeStore { - - final IntArrayList list; - - IntAttributeStore(AttributeInfo attrInfo) { - super(attrInfo); - this.list = new IntArrayList(); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.ints[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.ints[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.ints[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.ints[colPos] = (Integer) val; - } - - @Override - public void ensureCapacity(int pos) throws RepositoryException { - list.size(pos + 1); - nullList.size(pos + 1); - } - } - - static class LongAttributeStore extends PrimitiveAttributeStore { - - final LongArrayList list; - - LongAttributeStore(AttributeInfo attrInfo) { - super(attrInfo); - this.list = new LongArrayList(); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.longs[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.longs[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.longs[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.longs[colPos] = (Long) val; - } - - @Override - public void ensureCapacity(int pos) throws RepositoryException { - list.size(pos + 1); - nullList.size(pos + 1); - } - } - - static class FloatAttributeStore extends PrimitiveAttributeStore { - - final FloatArrayList list; - - FloatAttributeStore(AttributeInfo attrInfo) { - super(attrInfo); - this.list = new FloatArrayList(); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.floats[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.floats[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.floats[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.floats[colPos] = (Float) val; - } - - @Override - public void ensureCapacity(int pos) throws RepositoryException { - list.size(pos + 1); - nullList.size(pos + 1); - } - } - - static class DoubleAttributeStore extends PrimitiveAttributeStore { - - final DoubleArrayList list; - - DoubleAttributeStore(AttributeInfo attrInfo) { - super(attrInfo); - this.list = new DoubleArrayList(); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.doubles[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.doubles[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.doubles[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.doubles[colPos] = (Double) val; - } - - @Override - public void ensureCapacity(int pos) throws RepositoryException { - list.size(pos + 1); - nullList.size(pos + 1); - } - } - - static abstract class ObjectAttributeStore extends AbstractAttributeStore { - - final ArrayList list; - - ObjectAttributeStore(Class cls, AttributeInfo attrInfo) { - super(attrInfo); - this.list = Lists.newArrayList((T) null); - } - - @Override - public void ensureCapacity(int pos) throws RepositoryException { - while (list.size() < pos + 1) { - list.add((T) null); - } - nullList.size(pos + 1); - } - } - - static class BigIntStore extends ObjectAttributeStore { - - public BigIntStore(AttributeInfo attrInfo) { - super(BigInteger.class, attrInfo); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.bigIntegers[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.bigIntegers[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.bigIntegers[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.bigIntegers[colPos] = (BigInteger) val; - } - - } - - static class BigDecimalStore extends ObjectAttributeStore { - - public BigDecimalStore(AttributeInfo attrInfo) { - super(BigDecimal.class, attrInfo); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.bigDecimals[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.bigDecimals[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.bigDecimals[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.bigDecimals[colPos] = (BigDecimal) val; - } - - } - - static class DateStore extends ObjectAttributeStore { - - public DateStore(AttributeInfo attrInfo) { - super(Date.class, attrInfo); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.dates[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.dates[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.dates[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.dates[colPos] = (Date) val; - } - - } - - static class StringStore extends ObjectAttributeStore { - - public StringStore(AttributeInfo attrInfo) { - super(String.class, attrInfo); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.strings[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.strings[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.strings[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.strings[colPos] = (String) val; - } - - } - - static class IdStore extends ObjectAttributeStore { - - public IdStore(AttributeInfo attrInfo) { - super(Id.class, attrInfo); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.ids[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.ids[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.ids[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.ids[colPos] = (Id) val; - } - - } - - static class ImmutableListStore extends ObjectAttributeStore { - - public ImmutableListStore(AttributeInfo attrInfo) { - super(ImmutableList.class, attrInfo); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.arrays[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.arrays[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.arrays[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.arrays[colPos] = (ImmutableList) val; - } - - } - - static class ImmutableMapStore extends ObjectAttributeStore { - - public ImmutableMapStore(AttributeInfo attrInfo) { - super(ImmutableMap.class, attrInfo); - } - - protected void store(StructInstance instance, int colPos, int pos) { - list.set(pos, instance.maps[colPos]); - } - - protected void load(StructInstance instance, int colPos, int pos) { - instance.maps[colPos] = list.get(pos); - } - - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.maps[colPos]); - } - - protected void load(StructInstance instance, int colPos, Object val) { - instance.maps[colPos] = (ImmutableMap) val; - } - - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ClassStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ClassStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ClassStore.java deleted file mode 100755 index 465aa12..0000000 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ClassStore.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.repository.memory; - -import com.google.common.collect.ImmutableList; -import org.apache.hadoop.metadata.MetadataException; -import org.apache.hadoop.metadata.repository.RepositoryException; -import org.apache.hadoop.metadata.typesystem.persistence.Id; -import org.apache.hadoop.metadata.typesystem.persistence.ReferenceableInstance; -import org.apache.hadoop.metadata.typesystem.types.ClassType; - -import java.util.ArrayList; - -public class ClassStore extends HierarchicalTypeStore { - - final ArrayList> traitNamesStore; - final ClassType classType; - - public ClassStore(MemRepository repository, ClassType hierarchicalType) - throws RepositoryException { - super(repository, hierarchicalType); - classType = hierarchicalType; - traitNamesStore = new ArrayList>(); - } - - void store(ReferenceableInstance i) throws RepositoryException { - super.store(i); - int pos = idPosMap.get(i.getId()); - traitNamesStore.set(pos, i.getTraits()); - } - - public void ensureCapacity(int pos) throws RepositoryException { - super.ensureCapacity(pos); - while (traitNamesStore.size() < pos + 1) { - traitNamesStore.add(null); - } - } - - boolean validate(MemRepository repo, Id id) throws RepositoryException { - if (id.isUnassigned()) { - throw new RepositoryException(String.format("Invalid Id (unassigned) : %s", id)); - } - Integer pos = idPosMap.get(id); - if (pos == null) { - throw new RepositoryException(String.format("Invalid Id (unknown) : %s", id)); - } - - String typeName = typeNameList.get(pos); - if (typeName != hierarchicalType.getName()) { - throw new RepositoryException( - String.format("Invalid Id (incorrect typeName, type is %s) : %s", - typeName, id)); - } - - return true; - } - - /* - * - assumes id is already validated - */ - ReferenceableInstance createInstance(MemRepository repo, Id id) throws RepositoryException { - Integer pos = idPosMap.get(id); - String typeName = typeNameList.get(pos); - if (typeName != hierarchicalType.getName()) { - return repo.getClassStore(typeName).createInstance(repo, id); - } - - ImmutableList traitNames = traitNamesStore.get(pos); - String[] tNs = traitNames.toArray(new String[]{}); - - try { - ReferenceableInstance r = (ReferenceableInstance) classType.createInstance(id, tNs); - return r; - } catch (MetadataException me) { - throw new RepositoryException(me); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/HierarchicalTypeStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/HierarchicalTypeStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/HierarchicalTypeStore.java deleted file mode 100755 index 118f7fa..0000000 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/HierarchicalTypeStore.java +++ /dev/null @@ -1,209 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.repository.memory; - -import com.google.common.collect.ImmutableBiMap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import org.apache.hadoop.metadata.repository.RepositoryException; -import org.apache.hadoop.metadata.typesystem.persistence.Id; -import org.apache.hadoop.metadata.typesystem.persistence.ReferenceableInstance; -import org.apache.hadoop.metadata.typesystem.persistence.StructInstance; -import org.apache.hadoop.metadata.typesystem.types.AttributeInfo; -import org.apache.hadoop.metadata.typesystem.types.HierarchicalType; -import org.apache.hadoop.metadata.typesystem.types.IConstructableType; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -public abstract class HierarchicalTypeStore { - - final MemRepository repository; - final IConstructableType hierarchicalType; - final ArrayList typeNameList; - final ImmutableMap attrStores; - final ImmutableList superTypeStores; - - - /** - * Map Id to position in storage lists. - */ - Map idPosMap; - - List freePositions; - - int nextPos; - - /** - * Lock for each Class/Trait. - */ - ReentrantReadWriteLock lock; - - HierarchicalTypeStore(MemRepository repository, HierarchicalType hierarchicalType) - throws RepositoryException { - this.hierarchicalType = (IConstructableType) hierarchicalType; - this.repository = repository; - ImmutableMap.Builder b - = new ImmutableBiMap.Builder(); - typeNameList = Lists.newArrayList((String) null); - ImmutableList l = hierarchicalType.immediateAttrs; - for (AttributeInfo i : l) { - b.put(i, AttributeStores.createStore(i)); - } - attrStores = b.build(); - - ImmutableList.Builder b1 - = new ImmutableList.Builder(); - Set allSuperTypeNames = hierarchicalType.getAllSuperTypeNames(); - for (String s : allSuperTypeNames) { - b1.add(repository.getStore(s)); - } - superTypeStores = b1.build(); - - nextPos = 0; - idPosMap = new HashMap(); - freePositions = new ArrayList(); - - lock = new ReentrantReadWriteLock(); - } - - /** - * Assign a storage position to an Id. - * - try to assign from freePositions - * - ensure storage capacity. - * - add entry in idPosMap. - * @param id - * @return - * @throws RepositoryException - */ - int assignPosition(Id id) throws RepositoryException { - - int pos = -1; - if (!freePositions.isEmpty()) { - pos = freePositions.remove(0); - } else { - pos = nextPos++; - ensureCapacity(pos); - } - - idPosMap.put(id, pos); - - for (HierarchicalTypeStore s : superTypeStores) { - s.assignPosition(id); - } - - return pos; - } - - /** - * - remove from idPosMap - * - add to freePositions. - * @throws RepositoryException - */ - void releaseId(Id id) { - - Integer pos = idPosMap.get(id); - if (pos != null) { - idPosMap.remove(id); - freePositions.add(pos); - - for (HierarchicalTypeStore s : superTypeStores) { - s.releaseId(id); - } - } - } - - void acquireReadLock() { - lock.readLock().lock(); - } - - void acquireWriteLock() { - lock.writeLock().lock(); - } - - void releaseReadLock() { - lock.readLock().unlock(); - } - - void releaseWriteLock() { - lock.writeLock().unlock(); - } - - protected void storeFields(int pos, StructInstance s) throws RepositoryException { - for (Map.Entry e : attrStores.entrySet()) { - IAttributeStore attributeStore = e.getValue(); - attributeStore.store(pos, hierarchicalType, s); - } - } - - protected void loadFields(int pos, StructInstance s) throws RepositoryException { - for (Map.Entry e : attrStores.entrySet()) { - IAttributeStore attributeStore = e.getValue(); - attributeStore.load(pos, hierarchicalType, s); - } - } - - /** - * - store the typeName - * - store the immediate attributes in the respective IAttributeStore - * - call store on each SuperType. - * @param i - * @throws RepositoryException - */ - void store(ReferenceableInstance i) throws RepositoryException { - int pos = idPosMap.get(i.getId()); - typeNameList.set(pos, i.getTypeName()); - storeFields(pos, i); - - for (HierarchicalTypeStore s : superTypeStores) { - s.store(i); - } - } - - /** - * - copy over the immediate attribute values from the respective IAttributeStore - * - call load on each SuperType. - * @param i - * @throws RepositoryException - */ - void load(ReferenceableInstance i) throws RepositoryException { - int pos = idPosMap.get(i.getId()); - loadFields(pos, i); - - for (HierarchicalTypeStore s : superTypeStores) { - s.load(i); - } - } - - public void ensureCapacity(int pos) throws RepositoryException { - while (typeNameList.size() < pos + 1) { - typeNameList.add(null); - } - for (Map.Entry e : attrStores.entrySet()) { - IAttributeStore attributeStore = e.getValue(); - attributeStore.ensureCapacity(pos); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/IAttributeStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/IAttributeStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/IAttributeStore.java deleted file mode 100755 index 9a3f05c..0000000 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/IAttributeStore.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.repository.memory; - -import org.apache.hadoop.metadata.repository.RepositoryException; -import org.apache.hadoop.metadata.typesystem.persistence.StructInstance; -import org.apache.hadoop.metadata.typesystem.types.IConstructableType; - -public interface IAttributeStore { - /** - * Store the attribute's value from the 'instance' into this store. - * @param pos - * @param instance - * @throws RepositoryException - */ - void store(int pos, IConstructableType type, StructInstance instance) - throws RepositoryException; - - /** - * load the Instance with the value from position 'pos' for the attribute. - * @param pos - * @param instance - * @throws RepositoryException - */ - void load(int pos, IConstructableType type, StructInstance instance) throws RepositoryException; - - /** - * Ensure store have space for the given pos. - * @param pos - * @throws RepositoryException - */ - void ensureCapacity(int pos) throws RepositoryException; -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/MemRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/MemRepository.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/MemRepository.java deleted file mode 100755 index 74deccb..0000000 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/MemRepository.java +++ /dev/null @@ -1,329 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.repository.memory; - -import org.apache.hadoop.metadata.MetadataException; -import org.apache.hadoop.metadata.repository.DiscoverInstances; -import org.apache.hadoop.metadata.repository.IRepository; -import org.apache.hadoop.metadata.repository.RepositoryException; -import org.apache.hadoop.metadata.typesystem.IReferenceableInstance; -import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance; -import org.apache.hadoop.metadata.typesystem.persistence.Id; -import org.apache.hadoop.metadata.typesystem.persistence.MapIds; -import org.apache.hadoop.metadata.typesystem.persistence.ReferenceableInstance; -import org.apache.hadoop.metadata.typesystem.types.ClassType; -import org.apache.hadoop.metadata.typesystem.types.DataTypes; -import org.apache.hadoop.metadata.typesystem.types.HierarchicalType; -import org.apache.hadoop.metadata.typesystem.types.Multiplicity; -import org.apache.hadoop.metadata.typesystem.types.ObjectGraphWalker; -import org.apache.hadoop.metadata.typesystem.types.TraitType; -import org.apache.hadoop.metadata.typesystem.types.TypeSystem; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicInteger; - -public class MemRepository implements IRepository { - -/* - public static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - public static SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd"); -*/ - - final TypeSystem typeSystem; - /* - * A Store for each Class and Trait. - */ - final Map typeStores; - final AtomicInteger ID_SEQ = new AtomicInteger(0); - - public MemRepository(TypeSystem typeSystem) { - this.typeSystem = typeSystem; - this.typeStores = new HashMap<>(); - } - -/* - @Override - public DateFormat getDateFormat() { - return dateFormat; - } - - @Override - public DateFormat getTimestampFormat() { - return timestampFormat; - } - - @Override - public boolean allowNullsInCollections() { - return false; - } -*/ - - @Override - public Id newId(String typeName) { - return new Id("" + ID_SEQ.incrementAndGet(), 0, typeName); - } - - /** - * 1. traverse the Object Graph from i and create idToNewIdMap : Map[Id, Id], - * also create old Id to Instance Map: oldIdToInstance : Map[Id, IInstance] - * - traverse reference Attributes, List[ClassType], Maps where Key/value is ClassType - * - traverse Structs - * - traverse Traits. - * 1b. Ensure that every newId has an associated Instance. - * 2. Traverse oldIdToInstance map create newInstances : List[ITypedReferenceableInstance] - * - create a ITypedReferenceableInstance. - * replace any old References ( ids or object references) with new Ids. - * 3. Traverse over newInstances - * - ask ClassStore to assign a position to the Id. - * - for Instances with Traits, assign a position for each Trait - * - invoke store on the nwInstance. - * - * Recovery: - * - on each newInstance, invoke releaseId and delete on its ClassStore and Traits' Stores. - * - * @param i - * @return - * @throws org.apache.hadoop.metadata.repository.RepositoryException - */ - public ITypedReferenceableInstance create(IReferenceableInstance i) throws RepositoryException { - - DiscoverInstances discoverInstances = new DiscoverInstances(this); - - /* - * Step 1: traverse the Object Graph from i and create idToNewIdMap : Map[Id, Id], - * also create old Id to Instance Map: oldIdToInstance : Map[Id, IInstance] - * - traverse reference Attributes, List[ClassType], Maps where Key/value is ClassType - * - traverse Structs - * - traverse Traits. - */ - try { - new ObjectGraphWalker(typeSystem, discoverInstances, i).walk(); - } catch (MetadataException me) { - throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me); - } - - /* - * Step 1b: Ensure that every newId has an associated Instance. - */ - for (Id oldId : discoverInstances.idToNewIdMap.keySet()) { - if (!discoverInstances.idToInstanceMap.containsKey(oldId)) { - throw new RepositoryException(String.format("Invalid Object Graph: " + - "Encountered an unassignedId %s that is not associated with an Instance", - oldId)); - } - } - - /* Step 2: Traverse oldIdToInstance map create newInstances : - List[ITypedReferenceableInstance] - * - create a ITypedReferenceableInstance. - * replace any old References ( ids or object references) with new Ids. - */ - List newInstances - = new ArrayList(); - ITypedReferenceableInstance retInstance = null; - Set classTypes = new TreeSet(); - Set traitTypes = new TreeSet(); - for (IReferenceableInstance transientInstance : discoverInstances.idToInstanceMap - .values()) { - try { - ClassType cT = typeSystem - .getDataType(ClassType.class, transientInstance.getTypeName()); - ITypedReferenceableInstance newInstance = cT - .convert(transientInstance, Multiplicity.REQUIRED); - newInstances.add(newInstance); - - classTypes.add(cT); - for (String traitName : newInstance.getTraits()) { - TraitType tT = typeSystem.getDataType(TraitType.class, traitName); - traitTypes.add(tT); - } - - if (newInstance.getId() == i.getId()) { - retInstance = newInstance; - } - - /* - * Now replace old references with new Ids - */ - MapIds mapIds = new MapIds(discoverInstances.idToNewIdMap); - new ObjectGraphWalker(typeSystem, mapIds, newInstances).walk(); - - } catch (MetadataException me) { - throw new RepositoryException( - String.format("Failed to create Instance(id = %s", - transientInstance.getId()), me); - } - } - - /* - * 3. Acquire Class and Trait Storage locks. - * - acquire them in a stable order (super before subclass, classes before traits - */ - for (ClassType cT : classTypes) { - HierarchicalTypeStore st = typeStores.get(cT.getName()); - st.acquireWriteLock(); - } - - for (TraitType tT : traitTypes) { - HierarchicalTypeStore st = typeStores.get(tT.getName()); - st.acquireWriteLock(); - } - - - /* - * 4. Traverse over newInstances - * - ask ClassStore to assign a position to the Id. - * - for Instances with Traits, assign a position for each Trait - * - invoke store on the nwInstance. - */ - try { - for (ITypedReferenceableInstance instance : newInstances) { - HierarchicalTypeStore st = typeStores.get(instance.getTypeName()); - st.assignPosition(instance.getId()); - for (String traitName : instance.getTraits()) { - HierarchicalTypeStore tt = typeStores.get(traitName); - tt.assignPosition(instance.getId()); - } - } - - for (ITypedReferenceableInstance instance : newInstances) { - HierarchicalTypeStore st = typeStores.get(instance.getTypeName()); - st.store((ReferenceableInstance) instance); - for (String traitName : instance.getTraits()) { - HierarchicalTypeStore tt = typeStores.get(traitName); - tt.store((ReferenceableInstance) instance); - } - } - } catch (RepositoryException re) { - for (ITypedReferenceableInstance instance : newInstances) { - HierarchicalTypeStore st = typeStores.get(instance.getTypeName()); - st.releaseId(instance.getId()); - } - throw re; - } finally { - for (ClassType cT : classTypes) { - HierarchicalTypeStore st = typeStores.get(cT.getName()); - st.releaseWriteLock(); - } - - for (TraitType tT : traitTypes) { - HierarchicalTypeStore st = typeStores.get(tT.getName()); - st.releaseWriteLock(); - } - } - - return retInstance; - } - - public ITypedReferenceableInstance update(ITypedReferenceableInstance i) - throws RepositoryException { - throw new RepositoryException("not implemented"); - } - - public void delete(ITypedReferenceableInstance i) throws RepositoryException { - throw new RepositoryException("not implemented"); - } - - public ITypedReferenceableInstance get(Id id) throws RepositoryException { - - try { - ReplaceIdWithInstance replacer = new ReplaceIdWithInstance(this); - ObjectGraphWalker walker = new ObjectGraphWalker(typeSystem, replacer); - replacer.setWalker(walker); - ITypedReferenceableInstance r = getDuringWalk(id, walker); - walker.walk(); - return r; - } catch (MetadataException me) { - throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me); - } - } - - /* - * - Id must be valid; Class must be valid. - * - Ask ClassStore to createInstance. - * - Ask ClassStore to load instance. - * - load instance traits - * - add to GraphWalker - */ - ITypedReferenceableInstance getDuringWalk(Id id, ObjectGraphWalker walker) - throws RepositoryException { - ClassStore cS = getClassStore(id.getTypeName()); - if (cS == null) { - throw new RepositoryException(String.format("Unknown Class %s", id.getTypeName())); - } - cS.validate(this, id); - ReferenceableInstance r = cS.createInstance(this, id); - cS.load(r); - for (String traitName : r.getTraits()) { - HierarchicalTypeStore tt = typeStores.get(traitName); - tt.load(r); - } - - walker.addRoot(r); - return r; - } - - HierarchicalTypeStore getStore(String typeName) { - return typeStores.get(typeName); - } - - ClassStore getClassStore(String typeName) { - return (ClassStore) getStore(typeName); - } - - public void defineClass(ClassType type) throws RepositoryException { - HierarchicalTypeStore s = new ClassStore(this, type); - typeStores.put(type.getName(), s); - } - - public void defineTrait(TraitType type) throws RepositoryException { - HierarchicalTypeStore s = new TraitStore(this, type); - typeStores.put(type.getName(), s); - } - - public void defineTypes(List types) throws RepositoryException { - List tTypes = new ArrayList(); - List cTypes = new ArrayList(); - - for (HierarchicalType h : types) { - if (h.getTypeCategory() == DataTypes.TypeCategory.TRAIT) { - tTypes.add((TraitType) h); - } else { - cTypes.add((ClassType) h); - } - } - - Collections.sort(tTypes); - Collections.sort(cTypes); - - for (TraitType tT : tTypes) { - defineTrait(tT); - } - - for (ClassType cT : cTypes) { - defineClass(cT); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ReplaceIdWithInstance.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ReplaceIdWithInstance.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ReplaceIdWithInstance.java deleted file mode 100755 index f366d24..0000000 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/ReplaceIdWithInstance.java +++ /dev/null @@ -1,143 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.repository.memory; - -import com.google.common.collect.ImmutableCollection; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import org.apache.hadoop.metadata.MetadataException; -import org.apache.hadoop.metadata.typesystem.ITypedReferenceableInstance; -import org.apache.hadoop.metadata.typesystem.persistence.Id; -import org.apache.hadoop.metadata.typesystem.types.DataTypes; -import org.apache.hadoop.metadata.typesystem.types.Multiplicity; -import org.apache.hadoop.metadata.typesystem.types.ObjectGraphWalker; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -public class ReplaceIdWithInstance implements ObjectGraphWalker.NodeProcessor { - - public final Map idToInstanceMap; - final MemRepository repository; - ObjectGraphWalker walker; - - public ReplaceIdWithInstance(MemRepository repository) { - this.repository = repository; - idToInstanceMap = new HashMap<>(); - } - - void setWalker(ObjectGraphWalker walker) { - this.walker = walker; - } - - @Override - public void processNode(ObjectGraphWalker.Node nd) throws MetadataException { - if (nd.attributeName == null) { - // do nothing - } else if (!nd.aInfo.isComposite || nd.value == null) { - // do nothing - } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) { - - if (nd.value != null && nd.value instanceof Id) { - Id id = (Id) nd.value; - ITypedReferenceableInstance r = getInstance(id); - nd.instance.set(nd.attributeName, r); - } - } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) { - DataTypes.ArrayType aT = (DataTypes.ArrayType) nd.aInfo.dataType(); - nd.instance.set(nd.attributeName, - convertToInstances((ImmutableCollection) nd.value, nd.aInfo.multiplicity, aT)); - } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) { - DataTypes.MapType mT = (DataTypes.MapType) nd.aInfo.dataType(); - nd.instance.set(nd.attributeName, - convertToInstances((ImmutableMap) nd.value, nd.aInfo.multiplicity, mT)); - } - } - - ImmutableCollection convertToInstances(ImmutableCollection val, - Multiplicity m, DataTypes.ArrayType arrType) - throws MetadataException { - - if (val == null || - arrType.getElemType().getTypeCategory() != DataTypes.TypeCategory.CLASS) { - return val; - } - - ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder() - : ImmutableList.builder(); - Iterator it = val.iterator(); - while (it.hasNext()) { - Object elem = it.next(); - if (elem instanceof Id) { - Id id = (Id) elem; - elem = getInstance(id); - } - - b.add(elem); - - } - return b.build(); - } - - ImmutableMap convertToInstances(ImmutableMap val, Multiplicity m, - DataTypes.MapType mapType) - throws MetadataException { - - if (val == null || - (mapType.getKeyType().getTypeCategory() != DataTypes.TypeCategory.CLASS && - mapType.getValueType().getTypeCategory() != DataTypes.TypeCategory.CLASS)) { - return val; - } - ImmutableMap.Builder b = ImmutableMap.builder(); - Iterator it = val.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry elem = it.next(); - Object oldKey = elem.getKey(); - Object oldValue = elem.getValue(); - Object newKey = oldKey; - Object newValue = oldValue; - - if (oldKey instanceof Id) { - Id id = (Id) elem; - ITypedReferenceableInstance r = getInstance(id); - } - - if (oldValue instanceof Id) { - Id id = (Id) elem; - ITypedReferenceableInstance r = getInstance(id); - } - - b.put(newKey, newValue); - } - return b.build(); - } - - ITypedReferenceableInstance getInstance(Id id) throws MetadataException { - - ITypedReferenceableInstance r = idToInstanceMap.get(id); - if (r == null) { - r = repository.get(id); - idToInstanceMap.put(id, r); - walker.addRoot(r); - } - return r; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/StructStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/StructStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/StructStore.java deleted file mode 100755 index 502c691..0000000 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/StructStore.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.repository.memory; - -import com.google.common.collect.ImmutableBiMap; -import com.google.common.collect.ImmutableMap; -import org.apache.hadoop.metadata.repository.RepositoryException; -import org.apache.hadoop.metadata.typesystem.persistence.StructInstance; -import org.apache.hadoop.metadata.typesystem.types.AttributeInfo; -import org.apache.hadoop.metadata.typesystem.types.StructType; - -import java.util.Collection; -import java.util.Map; - -public class StructStore extends AttributeStores.AbstractAttributeStore implements IAttributeStore { - - final StructType structType; - final ImmutableMap attrStores; - - StructStore(AttributeInfo aInfo) throws RepositoryException { - super(aInfo); - this.structType = (StructType) aInfo.dataType(); - ImmutableMap.Builder b = new ImmutableBiMap.Builder<>(); - Collection l = structType.fieldMapping.fields.values(); - for (AttributeInfo i : l) { - b.put(i, AttributeStores.createStore(i)); - } - attrStores = b.build(); - - } - - @Override - protected void store(StructInstance instance, int colPos, int pos) throws RepositoryException { - StructInstance s = instance.structs[colPos]; - for (Map.Entry e : attrStores.entrySet()) { - IAttributeStore attributeStore = e.getValue(); - attributeStore.store(pos, structType, s); - } - } - - @Override - protected void load(StructInstance instance, int colPos, int pos) throws RepositoryException { - StructInstance s = (StructInstance) structType.createInstance(); - instance.structs[colPos] = s; - for (Map.Entry e : attrStores.entrySet()) { - IAttributeStore attributeStore = e.getValue(); - attributeStore.load(pos, structType, s); - } - } - - @Override - protected void store(StructInstance instance, int colPos, String attrName, - Map m) { - m.put(attrName, instance.structs[colPos]); - } - - @Override - protected void load(StructInstance instance, int colPos, Object val) { - instance.structs[colPos] = (StructInstance) val; - } - - @Override - public void ensureCapacity(int pos) throws RepositoryException { - for (Map.Entry e : attrStores.entrySet()) { - IAttributeStore attributeStore = e.getValue(); - attributeStore.ensureCapacity(pos); - } - nullList.size(pos + 1); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/TraitStore.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/TraitStore.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/TraitStore.java deleted file mode 100755 index 69a1894..0000000 --- a/repository/src/main/java/org/apache/hadoop/metadata/repository/memory/TraitStore.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.repository.memory; - -import org.apache.hadoop.metadata.repository.RepositoryException; -import org.apache.hadoop.metadata.typesystem.persistence.ReferenceableInstance; -import org.apache.hadoop.metadata.typesystem.persistence.StructInstance; -import org.apache.hadoop.metadata.typesystem.types.TraitType; - -import java.util.ArrayList; - -public class TraitStore extends HierarchicalTypeStore { - - final ArrayList classNameStore; - - public TraitStore(MemRepository repository, TraitType hierarchicalType) - throws RepositoryException { - super(repository, hierarchicalType); - classNameStore = new ArrayList<>(); - } - - void store(ReferenceableInstance i) throws RepositoryException { - int pos = idPosMap.get(i.getId()); - StructInstance s = (StructInstance) i.getTrait(hierarchicalType.getName()); - super.storeFields(pos, s); - classNameStore.set(pos, i.getTypeName()); - } - - void load(ReferenceableInstance i) throws RepositoryException { - int pos = idPosMap.get(i.getId()); - StructInstance s = (StructInstance) i.getTrait(hierarchicalType.getName()); - super.loadFields(pos, s); - } - - public void ensureCapacity(int pos) throws RepositoryException { - super.ensureCapacity(pos); - while (classNameStore.size() < pos + 1) { - classNameStore.add(null); - } - } -}