falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From srik...@apache.org
Subject [1/2] git commit: FALCON-259 Design Transformation Interface. Contributed by Srikanth Sundarrajan
Date Wed, 29 Jan 2014 07:04:00 GMT
Updated Branches:
  refs/heads/master 2b25d3c78 -> 0d1855229


FALCON-259 Design Transformation Interface. Contributed by Srikanth Sundarrajan


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/4ffdaef8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/4ffdaef8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/4ffdaef8

Branch: refs/heads/master
Commit: 4ffdaef813edc1475e7827908d6fc4a4c44c1217
Parents: 2b25d3c
Author: srikanth.sundarrajan <sriksun@apache.org>
Authored: Wed Jan 29 12:32:55 2014 +0530
Committer: srikanth.sundarrajan <sriksun@apache.org>
Committed: Wed Jan 29 12:32:55 2014 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |   2 +
 addons/designer/core/pom.xml                    |  10 ++
 .../designer/configuration/Configuration.java   |  26 +++++
 .../apache/falcon/designer/primitive/Code.java  |  27 +++++
 .../primitive/CompilationException.java         |  60 ++++++++++
 .../falcon/designer/primitive/Message.java      |  67 +++++++++++
 .../falcon/designer/primitive/Primitive.java    | 114 +++++++++++++++++++
 .../falcon/designer/primitive/Transform.java    |  86 ++++++++++++++
 .../falcon/designer/schema/RelationalData.java  |  53 +++++++++
 .../designer/schema/RelationalSchema.java       |  84 ++++++++++++++
 .../falcon/designer/source/DataSource.java      |  29 +++++
 .../apache/falcon/designer/storage/Storage.java |  67 +++++++++++
 .../designer/storage/StorageException.java      |  63 ++++++++++
 .../falcon/designer/storage/Storeable.java      |  52 +++++++++
 .../apache/falcon/designer/storage/Version.java |  71 ++++++++++++
 .../designer/storage/VersionedStorage.java      | 111 ++++++++++++++++++
 addons/designer/pom.xml                         |  30 ++---
 17 files changed, 937 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ed6587a..3ca68e5 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -5,6 +5,8 @@ Trunk (Unreleased)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+    FALCON-259 Design Transformation Interface. (Srikanth Sundarrajan)
+
     FALCON-26 Pipeline Monitor addition. (Suhas Vasu via Shwetha GS)
 
     FALCON-254 Bootstrap designer module. (Srikanth Sundarrajan via Shwetha GS)

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/pom.xml
----------------------------------------------------------------------
diff --git a/addons/designer/core/pom.xml b/addons/designer/core/pom.xml
index 1b0b735..e5dc36d 100644
--- a/addons/designer/core/pom.xml
+++ b/addons/designer/core/pom.xml
@@ -34,9 +34,19 @@
 
     <dependencies>
         <dependency>
+            <groupId>org.apache.falcon</groupId>
+            <artifactId>falcon-client</artifactId>
+        </dependency>
+
+        <dependency>
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>
         </dependency>
+
+        <dependency>
+            <groupId>net.sourceforge.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+        </dependency>
     </dependencies>
 
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
new file mode 100644
index 0000000..dba908a
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.configuration;
+
+/**
+ * This is the configuration that primitives will use
+ * to configure their instance.
+ */
+public interface Configuration {
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Code.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Code.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Code.java
new file mode 100644
index 0000000..35eeeb1
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Code.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.primitive;
+
+/**
+ * For now a simple marker interface for holding source code
+ * corresponding to a primitive.
+ */
+public interface Code {
+    //TODO Details to be figured out as go along.
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/CompilationException.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/CompilationException.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/CompilationException.java
new file mode 100644
index 0000000..603225b
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/CompilationException.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.primitive;
+
+import java.util.Iterator;
+
+/**
+ * Checked excpetion that is thrown when there are one more
+ * issues with the primitive that is being compiled.
+ */
+public class CompilationException extends Exception implements Iterable<Message> {
+
+    private final String detailedMessage;
+
+    private final Iterable<Message> messages;
+    /**
+     * Construct default CompilationException with the messages (that
+     * may typically be returned by the validation phase).
+     *
+     * @param validationMessages - Iterable messages.
+     */
+    public CompilationException(Iterable<Message> validationMessages) {
+        StringBuilder buffer = new StringBuilder();
+        for (Message message : validationMessages) {
+            if (buffer.length() > 0) {
+                buffer.append('\n');
+            }
+            buffer.append(message);
+        }
+        detailedMessage = buffer.toString();
+        messages = validationMessages;
+    }
+
+    @Override
+    public String getMessage() {
+        return detailedMessage;
+    }
+
+
+    @Override
+    public Iterator<Message> iterator() {
+        return messages.iterator();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Message.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Message.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Message.java
new file mode 100644
index 0000000..e5a68a8
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Message.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.primitive;
+
+/**
+ * Messages are pieces of information that may be surfaced to the caller
+ * by any of the operation on the primitives. For ex. Compilation on the
+ * transformation or flow may return some messages, some of which may be
+ * serious errors or warnings.
+ */
+public class Message {
+
+    /**
+     * Message type that each message is associated with.
+     */
+    public enum Type {ERROR, WARNING, INFORMATION}
+
+    private final Type type;
+    private final String message;
+    private Object context;
+
+    public Message(Type messageType, String messageText) {
+        this.type = messageType;
+        this.message = messageText;
+    }
+
+    public Type getType() {
+        return type;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public Object getContext() {
+        return context;
+    }
+
+    public void setContext(Object context) {
+        this.context = context;
+    }
+
+    @Override
+    public String toString() {
+        return "Message{"
+                + "type=" + type
+                + ", message='" + message + '\''
+                + ", context=" + context
+                + '}';
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
new file mode 100644
index 0000000..aa2b988
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.primitive;
+
+import org.apache.falcon.designer.configuration.Configuration;
+import org.apache.falcon.designer.storage.Storeable;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/**
+ * All elements of the pipeline are essentially a primitive.
+ * These primitives only have life during the pipeline design time.
+ *
+ */
+public abstract class Primitive<T extends Primitive> implements Storeable {
+
+    protected Configuration configuration;
+
+    public void setConfiguration(@Nonnull Configuration config) {
+        configuration = config;
+    }
+
+    protected abstract T copy();
+
+    @Nonnull
+    public Configuration getConfiguration() {
+        return configuration;
+    }
+
+    /**
+     * Perform a validation to see if the primitive configuration
+     * is consistent with this primitive.
+     *
+     * @return - Messages that need to be sent as feedback from the validation.
+     * As long as one of the messages returned has a message type.
+     * Returns null if there are no messages to return from the validation.
+     * {@link Message.Type} is {@link Message.Type#ERROR}, the primitive cannot
+     * be compiled successfully.
+     */
+    @Nullable
+    public abstract Iterable<Message> validate();
+
+    /**
+     * Compile the primitive and generate corresponding binary/source code
+     * The compile method fails if there are one or more
+     * {@link Message.Type#ERROR} messages from the validation phase.
+     *
+     * @return - Code object generated for the primitive.
+     * @throws CompilationException - Compilation issues as returned by validate
+     * (if any of the {@link Message} is of type {@link Message.Type#ERROR}
+     */
+    @Nonnull
+    public Code compile() throws CompilationException {
+        Iterable<Message> validationMessages = validate();
+        boolean error = false;
+        if (validationMessages != null) {
+            for (Message message : validationMessages) {
+                if (message.getType() == Message.Type.ERROR) {
+                    error = true;
+                    break;
+                }
+            }
+        }
+        if (error) {
+            throw new CompilationException(validationMessages);
+        } else {
+            T optimized = optimize();
+            return optimized.doCompile();
+        }
+    }
+
+    /**
+     * This method is invoked only when the primitive is confirmed to be
+     * valid. This would generate binary or source code for this primitive
+     * and its configuration.
+     *
+     * @return - Code object generated by the primitive.
+     */
+    @Nonnull
+    protected abstract Code doCompile();
+
+    /**
+     * This method is invoked only when the primitive is confirmed to be
+     * valid. This would operate on the current instance and return back
+     * an optimized version of the same type.
+     *
+     * @return - Optimized object of the same type.
+     */
+    @Nonnull
+    @SuppressWarnings("unchecked")
+    public T optimize() {
+        T copy = copy();
+        return (T) copy.doOptimize();
+    }
+
+    protected abstract T doOptimize();
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
new file mode 100644
index 0000000..72cf988
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.primitive;
+
+import org.apache.falcon.designer.schema.RelationalData;
+
+import javax.annotation.Nonnull;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Transform is a foundational primitive in Falcon designer. All well
+ * understood data transformations are to be implemented as a Transform.
+ *
+ * A transform would typically consume one or more data inputs conforming
+ * to a schema and would produce one or more output typically with an uniform schema.
+ *
+ */
+public abstract class Transform extends Primitive<Transform> {
+
+    protected List<RelationalData> inputData;
+    protected List<RelationalData> outputData;
+
+    /** Empty constructor to be used only for deserialization
+     * and cloning. Not otherwise.
+     */
+    protected Transform() {
+    }
+
+    /**
+     * Setter typically used for deserialization & cloning.
+     *
+     * @param inputData - List of input data
+     */
+    protected void setInputData(@Nonnull List<RelationalData> inputData) {
+        this.inputData = inputData;
+    }
+
+    /**
+     * Setter typically used for deserialization & cloning.
+     *
+     * @param outputData - List of output data
+     */
+    protected void setOutputData(@Nonnull List<RelationalData> outputData) {
+        this.outputData = outputData;
+    }
+
+    /**
+     * Each Transform by default requires one or more input data sets
+     * and produces a single output data set.
+     *
+     * @param inData - List of input data sets for this transform
+     * @param outData - List of Output data produced by this transform
+     */
+    protected Transform(@Nonnull List<RelationalData> inData, @Nonnull List<RelationalData>
outData) {
+        inputData = Collections.unmodifiableList(inData);
+        outputData = Collections.unmodifiableList(outData);
+    }
+
+    @Nonnull
+    public List<RelationalData> getInputData() {
+        return inputData;
+    }
+
+    @Nonnull
+    public List<RelationalData> getOutputData() {
+        return outputData;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/schema/RelationalData.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/schema/RelationalData.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/schema/RelationalData.java
new file mode 100644
index 0000000..d930e40
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/schema/RelationalData.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.schema;
+
+import org.apache.falcon.designer.source.DataSource;
+import org.apache.falcon.designer.storage.Storage;
+import org.apache.falcon.designer.storage.StorageException;
+import org.apache.falcon.designer.storage.Storeable;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Relational Data marker for now.
+ */
+public class RelationalData implements Storeable {
+
+    //TODO To work out details as we go along.
+
+    public RelationalData(RelationalSchema relationalSchema, DataSource dataSource) {
+
+    }
+
+    @Override
+    public void store(@Nonnull Storage storage) throws StorageException {
+        //TODO
+    }
+
+    @Override
+    public void restore(@Nonnull Storage storage) throws StorageException {
+        //TODO
+    }
+
+    @Override
+    public void delete(@Nonnull Storage storage) throws StorageException {
+        //TODO
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/schema/RelationalSchema.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/schema/RelationalSchema.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/schema/RelationalSchema.java
new file mode 100644
index 0000000..f4f44d1
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/schema/RelationalSchema.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.schema;
+
+import org.apache.falcon.designer.storage.Storage;
+import org.apache.falcon.designer.storage.StorageException;
+import org.apache.falcon.designer.storage.Storeable;
+
+import javax.annotation.Nonnull;
+import java.sql.Types;
+
+/**
+ * Relational Schema allows data to be represented similar to a relational
+ * table comprising of rows and columns. Data types for each column is
+ * allowed to be both primitive and complex data types ranging from simple
+ * integer to complex structures such as {@link java.util.Map},
+ * {@link java.util.Set}, {@link java.util.List}, Thrift structure and
+ * Protobuf messages etc.
+ */
+public class RelationalSchema implements Storeable {
+
+    /**
+     * Gets the total number of columns present in the
+     * underlying data source that conforms to this schema.
+     *
+     * @return Total Number of columns.
+     */
+    public int getColumnCount() {
+        return -1;
+    }
+
+    /**
+     * Gets the designated alias of a given column number
+     * as visible in the underlying data source.
+     *
+     * @param columnNumber - Can range from 0 to totalColumns - 1
+     * @return Alias of the column
+     */
+    public String getColumnAlias(int columnNumber) {
+        return "";
+    }
+
+    /**
+     * Gets the column type of a given column as visible
+     * in the underlying data source.
+     *
+     * @param columnNumber - Can range from 0 to totalColumns - 1
+     * @return {@link java.sql.Types}
+     */
+    public int getColumnType(int columnNumber) {
+        return Types.NULL;
+    }
+
+    @Override
+    public void store(@Nonnull Storage storage) throws StorageException {
+        //TODO
+    }
+
+    @Override
+    public void restore(@Nonnull Storage storage) throws StorageException {
+        //TODO
+    }
+
+    @Override
+    public void delete(@Nonnull Storage storage) throws StorageException {
+        //TODO
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/source/DataSource.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/source/DataSource.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/source/DataSource.java
new file mode 100644
index 0000000..227277c
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/source/DataSource.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.source;
+
+/**
+ * Data source on where the data is located. Generally to indicate
+ * if the data is persisted, if so where or if the data is transient
+ * or temporary.
+ */
+public interface DataSource {
+
+    //TODO This is just place holder. Behaviors will be added as we expand on the design
further
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Storage.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Storage.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Storage.java
new file mode 100644
index 0000000..5b63b31
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Storage.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.storage;
+
+import javax.annotation.Nonnull;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * This is the storage on which entities can be persisted and restored from.
+ *
+ * On the storage entities are organized under a namespace. Namespace/entity combination
+ * is unique on the storage.
+ */
+public interface Storage {
+
+    /**
+     * Opens an existing entity under the namespace and provides a Stream view of
+     * that data for the consumer.
+     *
+     * @param namespace - Namespace under which the entity is stored.
+     * @param entity - Entity that is being opened/read.
+     * @return - InputStream
+     * @throws StorageException - If such an entity doesn't exist or has issues
+     * reading from the storage.
+     */
+    @Nonnull
+    InputStream open(@Nonnull String namespace, @Nonnull String entity) throws StorageException;
+
+    /**
+     * Creates / Updates a new entity under the namespace and provides a Stream to write
out the
+     * data. If entity already exists under the namespace, same is overwritten.
+     *
+     * @param namespace - Namespace under which the entity is stored.
+     * @param entity - Entity that is being created/updated.
+     * @return - OutputStream
+     * @throws StorageException - If it has issues accessing or writing to the storage.
+     */
+    @Nonnull
+    OutputStream create(@Nonnull String namespace, @Nonnull String entity) throws StorageException;
+
+    /**
+     * Deletes an entity under the namespace specified if it exists.
+     *
+     * @param namespace - Namespace under which the entity is stored.
+     * @param entity - Entity that is being deleted.
+     * @throws StorageException - If entity is missing or if there are issues while performing
the
+     * delete operation
+     */
+    void delete(@Nonnull String namespace, @Nonnull String entity) throws StorageException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/StorageException.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/StorageException.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/StorageException.java
new file mode 100644
index 0000000..c8c2f58
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/StorageException.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.storage;
+
+/**
+ * Checked Exception that the {@link org.apache.falcon.designer.storage.Storeable}
+ * throws when there is an issue with either storing or restoring contents
+ * for an object from persistent storage.
+ */
+public class StorageException extends Exception {
+
+    /**
+     * Constructs a default exception with no cause or message.
+     */
+    public StorageException() {
+        super();
+    }
+
+    /**
+     * Constructs an exception with a specific message.
+     *
+     * @param message - Message on the exception
+     */
+    public StorageException(String message) {
+        super(message);
+    }
+
+    /**
+     * Constructs an exception with a specific message and cause.
+     *
+     * @param message - Message on the exception
+     * @param cause - Underlying exception that resulted in this being thrown
+     */
+    public StorageException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    /**
+     * Constructs an exception with a cause and message is initialized
+     * to be same as that of the cause.
+     *
+     * @param cause - Underlying exception that resulted in this being thrown
+     */
+    public StorageException(Throwable cause) {
+        super(cause);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Storeable.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Storeable.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Storeable.java
new file mode 100644
index 0000000..384d17a
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Storeable.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.storage;
+
+import javax.annotation.Nonnull;
+
+/**
+ * All elements in the designer that need to be persisted to permanent
+ * storage need to implement this interface.
+ */
+public interface Storeable {
+
+    /**
+     * Store the current object onto the storage being passed.
+     *
+     * @param storage - Storage onto which the object will be persisted or stored.
+     * @throws StorageException - Any exception from the underlying storage.
+     */
+    void store(@Nonnull Storage storage) throws StorageException;
+
+    /**
+     * Restore onto the current object contents from the Storage.
+     *
+     * @param storage - Storage from where the object will be restored from.
+     * @throws StorageException - Any exception from the underlying storage.
+     */
+    void restore(@Nonnull Storage storage) throws StorageException;
+
+    /**
+     * Deletes the current object from the storage permanently.
+     *
+     * @param storage - Storage on which the object is stored, that needs to be deleted
+     * @throws StorageException - Any exception from the underlying storage.
+     */
+    void delete(@Nonnull Storage storage) throws StorageException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Version.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Version.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Version.java
new file mode 100644
index 0000000..35c2e86
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/Version.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.storage;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Version that represents a specific snapshot of an entity.
+ */
+public class Version implements Comparable<Version> {
+
+    private final int version;
+    private final long timeStamp;
+
+    public Version(int version, long timeStamp) {
+        this.version = version;
+        this.timeStamp = timeStamp;
+    }
+
+    public int getVersion() {
+        return version;
+    }
+
+    public long getTimeStamp() {
+        return timeStamp;
+    }
+
+    @Override
+    public int compareTo(@Nonnull Version that) {
+        return Integer.valueOf(this.version).compareTo(that.version);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        Version that = (Version) obj;
+
+        if (this.version != that.version) {
+            return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return version;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/VersionedStorage.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/VersionedStorage.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/VersionedStorage.java
new file mode 100644
index 0000000..7f5edc5
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/VersionedStorage.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.storage;
+
+import javax.annotation.Nonnull;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * This is the storage on which entities can be persisted and restored from for a
+ * specific version.
+ *
+ * On the storage entities are organized under a namespace. Namespace/entity combination
+ * is unique on the storage.
+ */
+public interface VersionedStorage  extends Storage {
+
+    /**
+     * Opens the latest version of the existing entity under the namespace and
+     * provides a Stream view of that data for the consumer.
+     *
+     * @param namespace - Namespace under which the entity is stored.
+     * @param entity - Entity that is being opened/read.
+     * @return - InputStream
+     * @throws StorageException - If such an entity doesn't exist or has issues
+     * reading from the storage.
+     */
+    @Override
+    @Nonnull
+    InputStream open(@Nonnull String namespace, @Nonnull String entity) throws StorageException;
+
+    /**
+     * Opens the latest version of the existing entity under the namespace and
+     * provides a Stream view of that data for the consumer.
+     *
+     * @param namespace - Namespace under which the entity is stored.
+     * @param entity - Entity that is being opened/read.
+     * @param version - Version of the entity that needs to be opened.
+     * @return - InputStream
+     * @throws StorageException - If such an entity/version doesn't exist or has issues
+     * reading from the storage.
+     */
+    @Nonnull
+    InputStream open(@Nonnull String namespace, @Nonnull String entity,
+                     @Nonnull Version version) throws StorageException;
+
+    /**
+     * Creates / Updates a new entity under the namespace and provides a Stream to write
out the
+     * data. If entity already exists under the namespace, a new version of the same is created.
+     *
+     * @param namespace - Namespace under which the entity is stored.
+     * @param entity - Entity that is being created/updated.
+     * @return - OutputStream
+     * @throws StorageException - If it has issues accessing or writing to the storage.
+     */
+
+    @Override
+    @Nonnull
+    OutputStream create(@Nonnull String namespace, @Nonnull String entity) throws StorageException;
+
+    /**
+     * Deletes the latest version of an entity under the namespace specified if it exists.
+     *
+     * @param namespace - Namespace under which the entity is stored.
+     * @param entity - Entity that is being deleted.
+     * @throws StorageException - If entity is missing or if there are issues while performing
the
+     * delete operation
+     */
+    @Override
+    void delete(@Nonnull String namespace, @Nonnull String entity) throws StorageException;
+
+    /**
+     * Deletes the latest version of an entity under the namespace specified if it exists.
+     *
+     * @param namespace - Namespace under which the entity is stored.
+     * @param entity - Entity that is being deleted.
+     * @param version - Version that is to be deleted.
+     * @throws StorageException - If entity/version is missing or if there are issues while
performing the
+     * delete operation
+     */
+    void delete(@Nonnull String namespace, @Nonnull String entity,
+                @Nonnull Version version) throws StorageException;
+
+    /**
+     * Retrieves an iterator over versions of the entity under the namespace specified.
+     *
+     * @param namespace - Namespace underwhich the entity is stored.
+     * @param entity - Entity that is stored, for which versions are sought.
+     * @return - Iterable {@link org.apache.falcon.designer.storage.Version}
+     * @throws StorageException - If entity version is missing or if there are issues while
retrieving
+     * the versions on the entity.
+     */
+    @Nonnull
+    Iterable<Version> versions(@Nonnull String namespace, @Nonnull String entity) throws
StorageException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4ffdaef8/addons/designer/pom.xml
----------------------------------------------------------------------
diff --git a/addons/designer/pom.xml b/addons/designer/pom.xml
index 9e6ed2f..592bd3c 100644
--- a/addons/designer/pom.xml
+++ b/addons/designer/pom.xml
@@ -376,7 +376,7 @@
 
             <dependency>
                 <groupId>net.sourceforge.findbugs</groupId>
-                <artifactId>annotations</artifactId>
+                <artifactId>jsr305</artifactId>
                 <version>1.3.2</version>
             </dependency>
         </dependencies>
@@ -567,20 +567,20 @@
                     <useEclipseDefaultExcludes>true</useEclipseDefaultExcludes>
                     <excludeSubProjects>true</excludeSubProjects>
                     <excludes>
-                        <exclude>*.txt</exclude>
-                        <exclude>.git</exclude>
-                        <exclude>.idea</exclude>
-                        <exclude>*.twiki</exclude>
-                        <exclude>*.iml</exclude>
-                        <exclude>target</exclude>
-                        <exclude>*.patch</exclude>
-                        <exclude>*.log</exclude>
-                        <exclude>logs</exclude>
-                        <exclude>.classpath</exclude>
-                        <exclude>.project</exclude>
-                        <exclude>.settings</exclude>
-                        <exclude>maven-eclipse.xml</exclude>
-                        <exclude>.externalToolBuilders</exclude>
+                        <exclude>**/*.txt</exclude>
+                        <exclude>**/.git</exclude>
+                        <exclude>.idea/**</exclude>
+                        <exclude>**/*.twiki</exclude>
+                        <exclude>**/*.iml</exclude>
+                        <exclude>**/target</exclude>
+                        <exclude>**/*.patch</exclude>
+                        <exclude>**/*.log</exclude>
+                        <exclude>**/logs</exclude>
+                        <exclude>**/.classpath</exclude>
+                        <exclude>**/.project</exclude>
+                        <exclude>**/.settings</exclude>
+                        <exclude>**/maven-eclipse.xml</exclude>
+                        <exclude>**/.externalToolBuilders</exclude>
                     </excludes>
                 </configuration>
                 <executions>


Mime
View raw message