falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shweth...@apache.org
Subject git commit: FALCON-588 Baselining designer code. Contributed by samar
Date Wed, 13 Aug 2014 08:06:04 GMT
Repository: incubator-falcon
Updated Branches:
  refs/heads/master 839d44d60 -> aee6d3bf8


FALCON-588 Baselining designer code. Contributed by samar


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/aee6d3bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/aee6d3bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/aee6d3bf

Branch: refs/heads/master
Commit: aee6d3bf8e63763ac7924d1f4320cf23e4e6ef3a
Parents: 839d44d
Author: Shwetha GS <shwetha.gs@inmobi.com>
Authored: Wed Aug 13 13:34:47 2014 +0530
Committer: Shwetha GS <shwetha.gs@inmobi.com>
Committed: Wed Aug 13 13:35:15 2014 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |   2 +
 addons/designer/actions/pom.xml                 |  14 ++-
 .../configuration/EmailActionConfiguration.java |  74 ++++++++++++
 .../designer/primitive/action/EmailAction.java  |  92 +++++++++++++++
 addons/designer/core/pom.xml                    |  66 +++++++----
 .../configuration/ActionConfiguration.java      |  32 ++++++
 .../designer/configuration/Configuration.java   |  63 ++++++++++-
 .../designer/configuration/FlowConfig.java      |  69 +++++++++++
 .../designer/configuration/SerdeException.java  |  61 ++++++++++
 .../configuration/TransformConfiguration.java   |  33 ++++++
 .../falcon/designer/primitive/Action.java       |  12 +-
 .../falcon/designer/primitive/Primitive.java    | 113 +++++++++++++------
 .../falcon/designer/primitive/Transform.java    |   4 +-
 .../designer/storage/impl/HDFSStorage.java      |  98 ++++++++++++++++
 .../designer/storage/impl/HDFSStorageTest.java  |  78 +++++++++++++
 addons/designer/flows/pom.xml                   |   4 +
 .../apache/falcon/designer/primitive/Flow.java  |  83 ++++++++++++++
 addons/designer/pom.xml                         |  47 +++++++-
 18 files changed, 867 insertions(+), 78 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 8ee08e8..63552b0 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -5,6 +5,8 @@ Trunk (Unreleased)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+   FALCON-588 Baselining designer code. (samar via Shwetha GS)
+
    FALCON-400 Add Authorization for Entities (Venkatesh Seetharam)
       FALCON-279 Add ACL for Cluster Entity
       FALCON-462 Add ACL for process entity

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/actions/pom.xml
----------------------------------------------------------------------
diff --git a/addons/designer/actions/pom.xml b/addons/designer/actions/pom.xml
index 7d5afbb..bba43fd 100644
--- a/addons/designer/actions/pom.xml
+++ b/addons/designer/actions/pom.xml
@@ -33,10 +33,14 @@
     <packaging>jar</packaging>
 
     <dependencies>
-        <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-        </dependency>
-    </dependencies>
+	<dependency>
+		<groupId>org.testng</groupId>
+		<artifactId>testng</artifactId>
+	</dependency>
+	<dependency>
+		<groupId>org.apache.falcon.designer</groupId>
+		<artifactId>designer-core</artifactId>
+	</dependency>
+</dependencies>
 
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/actions/src/main/java/org/apache/falcon/designer/action/configuration/EmailActionConfiguration.java
----------------------------------------------------------------------
diff --git a/addons/designer/actions/src/main/java/org/apache/falcon/designer/action/configuration/EmailActionConfiguration.java
b/addons/designer/actions/src/main/java/org/apache/falcon/designer/action/configuration/EmailActionConfiguration.java
new file mode 100644
index 0000000..fd37a49
--- /dev/null
+++ b/addons/designer/actions/src/main/java/org/apache/falcon/designer/action/configuration/EmailActionConfiguration.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.action.configuration;
+
+import org.apache.falcon.designer.configuration.ActionConfiguration;
+/**
+ * Action object holding Email data.
+ */
+public class EmailActionConfiguration extends ActionConfiguration<EmailActionConfiguration>
{
+
+    private String to;
+    private String cc;
+    private String subject;
+    private String body;
+
+    private static final String NAME = "Email";
+
+    public String getTo() {
+        return to;
+    }
+
+    public void setTo(String to) {
+        this.to = to;
+    }
+
+    public String getCc() {
+        return cc;
+    }
+
+    public void setCc(String cc) {
+        this.cc = cc;
+    }
+
+    public String getSubject() {
+        return subject;
+    }
+
+    public void setSubject(String subject) {
+        this.subject = subject;
+    }
+
+    public String getBody() {
+        return body;
+    }
+
+    public void setBody(String body) {
+        this.body = body;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public Class<EmailActionConfiguration> getConfigClass() {
+        return EmailActionConfiguration.class;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/actions/src/main/java/org/apache/falcon/designer/primitive/action/EmailAction.java
----------------------------------------------------------------------
diff --git a/addons/designer/actions/src/main/java/org/apache/falcon/designer/primitive/action/EmailAction.java
b/addons/designer/actions/src/main/java/org/apache/falcon/designer/primitive/action/EmailAction.java
new file mode 100644
index 0000000..1bd1197
--- /dev/null
+++ b/addons/designer/actions/src/main/java/org/apache/falcon/designer/primitive/action/EmailAction.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.primitive.action;
+
+import org.apache.falcon.designer.action.configuration.EmailActionConfiguration;
+import org.apache.falcon.designer.primitive.Action;
+import org.apache.falcon.designer.primitive.Code;
+import org.apache.falcon.designer.primitive.Message;
+
+/**
+ * EmailAction Primitive containing implementation to compile.
+ */
+public class EmailAction extends Action<EmailAction, EmailActionConfiguration> {
+
+    private String nameSpace;
+    private String entity;
+
+    private EmailActionConfiguration emailConfig;
+
+    public EmailAction(EmailActionConfiguration config , String nameSpace, String entity)
{
+        this.emailConfig = config;
+        this.nameSpace = nameSpace;
+        this.entity = entity;
+    }
+
+    @Override
+    public EmailActionConfiguration getConfiguration() {
+        return emailConfig;
+    }
+
+    @Override
+    public void setConfiguration(EmailActionConfiguration config) {
+        this.emailConfig = config;
+
+    }
+
+    @Override
+    public boolean hasOutput() {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
+    @Override
+    protected EmailAction copy() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Iterable<Message> validate() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    protected Code doCompile() {
+        getConfiguration().getBody();
+        return null;
+    }
+
+    @Override
+    protected EmailAction doOptimize() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public String getNamespace() {
+        return nameSpace;
+    }
+
+    @Override
+    public String getEntity() {
+        return entity;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/pom.xml
----------------------------------------------------------------------
diff --git a/addons/designer/core/pom.xml b/addons/designer/core/pom.xml
index ddd8814..8ebc79a 100644
--- a/addons/designer/core/pom.xml
+++ b/addons/designer/core/pom.xml
@@ -32,26 +32,50 @@
     <name>Apache Falcon Designer Core</name>
     <packaging>jar</packaging>
 
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.falcon</groupId>
-            <artifactId>falcon-client</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>net.sourceforge.findbugs</groupId>
-            <artifactId>jsr305</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>commons-lang</groupId>
-            <artifactId>commons-lang</artifactId>
-        </dependency>
-    </dependencies>
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.falcon</groupId>
+			<artifactId>falcon-client</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>org.testng</groupId>
+			<artifactId>testng</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>net.sourceforge.findbugs</groupId>
+			<artifactId>jsr305</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>commons-lang</groupId>
+			<artifactId>commons-lang</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.testng</groupId>
+			<artifactId>testng</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-common</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-hdfs</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-hdfs</artifactId>
+			<scope>test</scope>
+			<classifier>tests</classifier>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-common</artifactId>
+			<scope>test</scope>
+			<classifier>tests</classifier>
+		</dependency>
+	</dependencies>
 
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/ActionConfiguration.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/ActionConfiguration.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/ActionConfiguration.java
new file mode 100644
index 0000000..884f459
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/ActionConfiguration.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.configuration;
+
+/**
+ * Abstract ActionConfiguration extending Configuration.
+ */
+public abstract class ActionConfiguration<A extends ActionConfiguration> extends Configuration<A>
{
+
+    private static final String CATEGORY = "ACTION";
+
+    @Override
+    public String getCategory() {
+        return CATEGORY;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
index dba908a..30c0314 100644
--- a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
@@ -15,12 +15,67 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.falcon.designer.configuration;
 
+import java.io.IOException;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+
 /**
- * This is the configuration that primitives will use
- * to configure their instance.
+ * This is the configuration that primitives will use to configure their
+ * instance.
  */
-public interface Configuration {
+public abstract class Configuration<T extends Configuration> {
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+
+    public abstract String getName();
+
+    public abstract String getCategory();
+
+    /**
+     * Serialize it from a action type to string type.
+     * @param act
+     *            actual Action Configuration data
+     * @return
+     */
+    public String serialize() throws SerdeException {
+        String returnJsonString;
+        try {
+            returnJsonString = MAPPER.writeValueAsString(this);
+        } catch (JsonGenerationException e) {
+            throw new SerdeException("Failed serializing object ", e);
+        } catch (JsonMappingException e) {
+            throw new SerdeException("Failed serializing object ", e);
+        } catch (IOException e) {
+            throw new SerdeException("Failed serializing object ", e);
+        }
+        return returnJsonString;
+    }
+
+    /**
+     * Deserialize from string to object of Action type.
+     * @param actString
+     *            actual Data
+     * @return
+     */
+    public T deserialize(String actString) throws SerdeException {
+        T returnEmailAction;
+        try {
+            returnEmailAction = MAPPER.readValue(actString, getConfigClass());
+        } catch (JsonParseException e) {
+            throw new SerdeException("Failed deserialize string " + actString,
+                e);
+        } catch (JsonMappingException e) {
+            throw new SerdeException("Failed deserialize string " + actString,
+                e);
+        } catch (IOException e) {
+            throw new SerdeException("Failed deserialize string " + actString,
+                e);
+        }
+        return returnEmailAction;
+    }
+
+    public abstract Class<T> getConfigClass();
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/FlowConfig.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/FlowConfig.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/FlowConfig.java
new file mode 100644
index 0000000..ba7d074
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/FlowConfig.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.configuration;
+
+/**
+ * Flow data. More parameters would need to be added.
+ */
+public class FlowConfig extends Configuration<FlowConfig> {
+
+    private String namespace;
+    private String entity;
+    private String name;
+    private static final String CATEGORY = "FLOW";
+
+    public FlowConfig(String namespace, String entity, String name) {
+        this.namespace = namespace;
+        this.entity = entity;
+        this.name = name;
+
+    }
+
+    @Override
+    public String getCategory() {
+        return CATEGORY;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    public String getNamespace() {
+        return namespace;
+    }
+
+    public void setNamespace(String namespace) {
+        this.namespace = namespace;
+    }
+
+    public String getEntity() {
+        return entity;
+    }
+
+    public void setEntity(String entity) {
+        this.entity = entity;
+    }
+
+    @Override
+    public Class<FlowConfig> getConfigClass() {
+        return FlowConfig.class;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/SerdeException.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/SerdeException.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/SerdeException.java
new file mode 100644
index 0000000..107180c
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/SerdeException.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.configuration;
+
+/**
+ * Checked Exception that is thrown on serialization/deserialization of a config
+ * object failure.
+ */
+public class SerdeException extends Exception {
+    /**
+     * Constructs a default exception with no cause or message.
+     */
+    public SerdeException() {
+        super();
+    }
+
+    /**
+     * Constructs an exception with a specific message.
+     * @param message
+     *            - Message on the exception
+     */
+    public SerdeException(String message) {
+        super(message);
+    }
+
+    /**
+     * Constructs an exception with a specific message and cause.
+     * @param message
+     *            - Message on the exception
+     * @param cause
+     *            - Underlying exception that resulted in this being thrown
+     */
+    public SerdeException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    /**
+     * Constructs an exception with a cause and message is initialized to be
+     * same as that of the cause.
+     * @param cause
+     *            - Underlying exception that resulted in this being thrown
+     */
+    public SerdeException(Throwable cause) {
+        super(cause);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/TransformConfiguration.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/TransformConfiguration.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/TransformConfiguration.java
new file mode 100644
index 0000000..6e0fedb
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/TransformConfiguration.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.configuration;
+
+/**
+ * Concrete implementation of Transform Configuration.
+ */
+public abstract class TransformConfiguration<A extends TransformConfiguration> extends
Configuration<A> {
+
+
+    private static final String CATEGORY = "TRANSFORM";
+
+    @Override
+    public String getCategory() {
+        return CATEGORY;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Action.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Action.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Action.java
index c40e462..e16c013 100644
--- a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Action.java
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Action.java
@@ -19,9 +19,11 @@
 package org.apache.falcon.designer.primitive;
 
 import org.apache.commons.lang.NotImplementedException;
+import org.apache.falcon.designer.configuration.ActionConfiguration;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+
 import java.net.URI;
 import java.util.Map;
 
@@ -39,17 +41,13 @@ import java.util.Map;
  * There are no restrictions imposed on an Action with respect to the input
  * or the output such as in a Transformation.
  */
-public abstract class Action extends Primitive {
-
-    protected final String type;
+public abstract class Action<A extends Action, B extends ActionConfiguration> extends
Primitive<A, B> {
 
-    protected Action(@Nonnull String type) {
-        this.type = type;
-    }
+    protected  static final String TYPE="ACTION";
 
     @Nonnull
     public String getType() {
-        return type;
+        return TYPE;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
index aa2b988..aa825a9 100644
--- a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
@@ -18,53 +18,53 @@
 
 package org.apache.falcon.designer.primitive;
 
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
 import org.apache.falcon.designer.configuration.Configuration;
+import org.apache.falcon.designer.configuration.SerdeException;
+import org.apache.falcon.designer.storage.Storage;
+import org.apache.falcon.designer.storage.StorageException;
 import org.apache.falcon.designer.storage.Storeable;
-
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 /**
- * All elements of the pipeline are essentially a primitive.
- * These primitives only have life during the pipeline design time.
- *
+ * All elements of the pipeline are essentially a primitive. These primitives
+ * only have life during the pipeline design time.
  */
-public abstract class Primitive<T extends Primitive> implements Storeable {
+public abstract class Primitive<T extends Primitive, V extends Configuration>
+    implements Storeable {
 
-    protected Configuration configuration;
-
-    public void setConfiguration(@Nonnull Configuration config) {
-        configuration = config;
-    }
+    public abstract void setConfiguration(V config);
 
     protected abstract T copy();
 
     @Nonnull
-    public Configuration getConfiguration() {
-        return configuration;
-    }
+    public abstract V getConfiguration();
 
     /**
-     * Perform a validation to see if the primitive configuration
-     * is consistent with this primitive.
-     *
+     * Perform a validation to see if the primitive configuration is consistent
+     * with this primitive.
      * @return - Messages that need to be sent as feedback from the validation.
-     * As long as one of the messages returned has a message type.
-     * Returns null if there are no messages to return from the validation.
-     * {@link Message.Type} is {@link Message.Type#ERROR}, the primitive cannot
-     * be compiled successfully.
+     *         As long as one of the messages returned has a message type.
+     *         Returns null if there are no messages to return from the
+     *         validation. {@link Message.Type} is {@link Message.Type#ERROR},
+     *         the primitive cannot be compiled successfully.
      */
     @Nullable
     public abstract Iterable<Message> validate();
 
     /**
-     * Compile the primitive and generate corresponding binary/source code
-     * The compile method fails if there are one or more
-     * {@link Message.Type#ERROR} messages from the validation phase.
-     *
+     * Compile the primitive and generate corresponding binary/source code The
+     * compile method fails if there are one or more {@link Message.Type#ERROR}
+     * messages from the validation phase.
      * @return - Code object generated for the primitive.
-     * @throws CompilationException - Compilation issues as returned by validate
-     * (if any of the {@link Message} is of type {@link Message.Type#ERROR}
+     * @throws CompilationException
+     *             - Compilation issues as returned by validate (if any of the
+     *             {@link Message} is of type {@link Message.Type#ERROR}
      */
     @Nonnull
     public Code compile() throws CompilationException {
@@ -87,20 +87,18 @@ public abstract class Primitive<T extends Primitive> implements
Storeable {
     }
 
     /**
-     * This method is invoked only when the primitive is confirmed to be
-     * valid. This would generate binary or source code for this primitive
-     * and its configuration.
-     *
+     * This method is invoked only when the primitive is confirmed to be valid.
+     * This would generate binary or source code for this primitive and its
+     * configuration.
      * @return - Code object generated by the primitive.
      */
     @Nonnull
     protected abstract Code doCompile();
 
     /**
-     * This method is invoked only when the primitive is confirmed to be
-     * valid. This would operate on the current instance and return back
-     * an optimized version of the same type.
-     *
+     * This method is invoked only when the primitive is confirmed to be valid.
+     * This would operate on the current instance and return back an optimized
+     * version of the same type.
      * @return - Optimized object of the same type.
      */
     @Nonnull
@@ -111,4 +109,51 @@ public abstract class Primitive<T extends Primitive> implements
Storeable {
     }
 
     protected abstract T doOptimize();
+
+    public abstract String getNamespace();
+
+    public abstract String getEntity();
+
+    @Override
+    public void store(Storage storage) throws StorageException {
+        try {
+
+            BufferedWriter bufferedWriterInst =
+                new BufferedWriter(new OutputStreamWriter(storage.create(
+                    getNamespace(), getEntity())));
+            String serializedResource =
+                getConfiguration().serialize();
+            bufferedWriterInst.write(serializedResource);
+            bufferedWriterInst.close();
+        } catch (IOException e) {
+            throw new StorageException(e.getMessage());
+        } catch (SerdeException e) {
+            throw new StorageException(e.getMessage());
+        }
+
+    }
+
+    @Override
+    public void restore(Storage storage) throws StorageException {
+        try {
+            BufferedReader bufferedReaderInst =
+                new BufferedReader(new InputStreamReader(storage.open(
+                    getNamespace(), getEntity())));
+            String configInString = bufferedReaderInst.readLine();
+            setConfiguration((V)getConfiguration().deserialize(configInString));
+            bufferedReaderInst.close();
+        } catch (IOException e) {
+            throw new StorageException(e.getMessage());
+        } catch (SerdeException e) {
+            throw new StorageException(e.getMessage());
+        }
+
+    }
+
+    @Override
+    public void delete(Storage storage) throws StorageException {
+        storage.delete(getNamespace(), getEntity());
+        setConfiguration(null);
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
index 72cf988..b2d4396 100644
--- a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
@@ -18,9 +18,11 @@
 
 package org.apache.falcon.designer.primitive;
 
+import org.apache.falcon.designer.configuration.TransformConfiguration;
 import org.apache.falcon.designer.schema.RelationalData;
 
 import javax.annotation.Nonnull;
+
 import java.util.Collections;
 import java.util.List;
 
@@ -32,7 +34,7 @@ import java.util.List;
  * to a schema and would produce one or more output typically with an uniform schema.
  *
  */
-public abstract class Transform extends Primitive<Transform> {
+public abstract class Transform extends Primitive<Transform , TransformConfiguration>
{
 
     protected List<RelationalData> inputData;
     protected List<RelationalData> outputData;

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/impl/HDFSStorage.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/impl/HDFSStorage.java
b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/impl/HDFSStorage.java
new file mode 100644
index 0000000..4d0ff64
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/impl/HDFSStorage.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.storage.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.falcon.designer.storage.Storage;
+import org.apache.falcon.designer.storage.StorageException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Storage implementation to store in HDFS.
+ *
+ */
+public class HDFSStorage implements Storage {
+
+    private FileSystem fs;
+    private String basePath;
+    private static final String SEPERATOR = "/";
+    private static final String BASEPATH_CONFIG_NAME =
+        "falcon.designer.hdfsstorage.defaultpath";
+
+    HDFSStorage(Configuration conf) throws StorageException {
+        try {
+            this.fs = FileSystem.get(conf);
+        } catch (IOException e) {
+            throw new StorageException(e);
+        }
+        this.basePath = conf.get(BASEPATH_CONFIG_NAME);
+        if (this.basePath == null || this.basePath.isEmpty()) {
+            throw new StorageException(BASEPATH_CONFIG_NAME
+                + " cannot be empty");
+        }
+
+    }
+
+    @Override
+    public InputStream open(String namespace, String entity)
+        throws StorageException {
+        try {
+            return fs.open(new Path(basePath + SEPERATOR + namespace
+                + SEPERATOR + entity));
+
+        } catch (IllegalArgumentException e) {
+            throw new StorageException(e);
+        } catch (IOException e) {
+            throw new StorageException(e);
+        }
+
+    }
+
+    @Override
+    public OutputStream create(String namespace, String entity)
+        throws StorageException {
+        try {
+            return fs.create(new Path(basePath + SEPERATOR + namespace
+                + SEPERATOR + entity));
+        } catch (IllegalArgumentException e) {
+            throw new StorageException(e);
+        } catch (IOException e) {
+            throw new StorageException(e);
+        }
+
+    }
+
+    @Override
+    public void delete(String namespace, String entity) throws StorageException {
+        try {
+            fs.delete(new Path(basePath + SEPERATOR + namespace + SEPERATOR
+                + entity), true);
+        } catch (IllegalArgumentException e) {
+            throw new StorageException(e);
+        } catch (IOException e) {
+            throw new StorageException(e);
+        }
+
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/test/java/org/apache/falcon/designer/storage/impl/HDFSStorageTest.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/test/java/org/apache/falcon/designer/storage/impl/HDFSStorageTest.java
b/addons/designer/core/src/test/java/org/apache/falcon/designer/storage/impl/HDFSStorageTest.java
new file mode 100644
index 0000000..187a7d2
--- /dev/null
+++ b/addons/designer/core/src/test/java/org/apache/falcon/designer/storage/impl/HDFSStorageTest.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.storage.impl;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.falcon.designer.storage.StorageException;
+import org.apache.hadoop.conf.Configuration;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Will test HDFSStorage for create open remove.
+ */
+public class HDFSStorageTest {
+    private HDFSStorage hdfsStorageInst;
+
+    @BeforeClass
+    public void setUpDFS() throws Exception {
+        Configuration conf = new Configuration();
+        conf.set("falcon.designer.hdfsstorage.defaultpath", "/tmp/");
+        conf.set("fs.default.name", "file:///");
+        hdfsStorageInst = new HDFSStorage(conf);
+
+    }
+
+    @Test
+    public void testCreateOpenDelete() {
+        try {
+            final String testNameSpace = "testNS";
+            final String testEntity = "testEntity";
+            OutputStream opStream =
+                    hdfsStorageInst.create(testNameSpace, testEntity);
+            String testMessage = "testing HDFSStorage";
+            byte[] outputByte = new byte[testMessage.length()];
+            opStream.write(testMessage.getBytes());
+            opStream.close();
+            InputStream ipStream =
+                    hdfsStorageInst.open(testNameSpace, testEntity);
+            ipStream.read(outputByte, 0, testMessage.length());
+            ipStream.close();
+            hdfsStorageInst.delete(testNameSpace, testEntity);
+            try {
+                hdfsStorageInst.open(testNameSpace, testEntity);
+                Assert
+                    .fail("file should be present and should have thrown an exception");
+            } catch (StorageException ex) {
+                Assert.assertEquals(ex.getCause().getClass(),
+                        FileNotFoundException.class);
+            }
+            Assert.assertEquals(new String(outputByte), testMessage);
+        } catch (StorageException ex) {
+            Assert.fail(ex.getMessage());
+        } catch (IOException ex) {
+            Assert.fail(ex.getMessage());
+        }
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/flows/pom.xml
----------------------------------------------------------------------
diff --git a/addons/designer/flows/pom.xml b/addons/designer/flows/pom.xml
index ce706a3..e47154f 100644
--- a/addons/designer/flows/pom.xml
+++ b/addons/designer/flows/pom.xml
@@ -37,6 +37,10 @@
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>
         </dependency>
+		<dependency>
+			<groupId>org.apache.falcon.designer</groupId>
+			<artifactId>designer-core</artifactId>
+		</dependency>
     </dependencies>
 
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/flows/src/main/java/org/apache/falcon/designer/primitive/Flow.java
----------------------------------------------------------------------
diff --git a/addons/designer/flows/src/main/java/org/apache/falcon/designer/primitive/Flow.java
b/addons/designer/flows/src/main/java/org/apache/falcon/designer/primitive/Flow.java
new file mode 100644
index 0000000..f7d227c
--- /dev/null
+++ b/addons/designer/flows/src/main/java/org/apache/falcon/designer/primitive/Flow.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.primitive;
+
+import org.apache.falcon.designer.configuration.FlowConfig;
+
+/**
+ * Concrete implementation for a Flow.
+ */
+public class Flow extends Primitive<Flow, FlowConfig> {
+
+    private FlowConfig process;
+    private String nameSpace;
+    private String entity;
+
+    public Flow(FlowConfig process, String nameSpace, String entity) {
+        this.process = process;
+        this.nameSpace = nameSpace;
+        this.entity = entity;
+    }
+
+    @Override
+    protected Flow copy() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Iterable<Message> validate() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    protected Code doCompile() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    protected Flow doOptimize() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public String getNamespace() {
+        return nameSpace;
+    }
+
+    @Override
+    public String getEntity() {
+        return entity;
+    }
+
+
+    @Override
+    public void setConfiguration(FlowConfig config) {
+        this.process = config;
+    }
+
+    @Override
+    public FlowConfig getConfiguration() {
+        return process;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/pom.xml
----------------------------------------------------------------------
diff --git a/addons/designer/pom.xml b/addons/designer/pom.xml
index 3e1a98a..2330682 100644
--- a/addons/designer/pom.xml
+++ b/addons/designer/pom.xml
@@ -102,6 +102,7 @@
         <jetty.version>6.1.26</jetty.version>
         <internal.maven.repo>file:///tmp/falcontemprepo</internal.maven.repo>
         <skipCheck>false</skipCheck>
+        <hadoop.version>2.2.0</hadoop.version>
     </properties>
 
     <modules>
@@ -368,11 +369,40 @@
                 <version>${hcatalog.version}</version>
             </dependency>
 
-            <dependency>
-                <groupId>net.sourceforge.findbugs</groupId>
-                <artifactId>jsr305</artifactId>
-                <version>1.3.2</version>
-            </dependency>
+			<dependency>
+				<groupId>net.sourceforge.findbugs</groupId>
+				<artifactId>jsr305</artifactId>
+				<version>1.3.2</version>
+			</dependency>
+			<dependency>
+				<groupId>org.apache.falcon.designer</groupId>
+				<artifactId>designer-core</artifactId>
+				<version>${project.version}</version>
+			</dependency>
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-common</artifactId>
+				<version>${hadoop.version}</version>
+			</dependency>
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-hdfs</artifactId>
+				<version>${hadoop.version}</version>
+			</dependency>
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-hdfs</artifactId>
+				<version>${hadoop.version}</version>
+				<scope>test</scope>
+				<classifier>tests</classifier>
+			</dependency>
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-common</artifactId>
+				<version>${hadoop.version}</version>
+				<scope>test</scope>
+				<classifier>tests</classifier>
+			</dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -575,6 +605,8 @@
                         <exclude>**/.settings</exclude>
                         <exclude>**/maven-eclipse.xml</exclude>
                         <exclude>**/.externalToolBuilders</exclude>
+                        <exclude>build/**</exclude>
+                        <exclude>test-output/**</exclude>
                     </excludes>
                 </configuration>
                 <executions>
@@ -584,6 +616,9 @@
                             <goal>check</goal>
                         </goals>
                         <phase>verify</phase>
+                        <configuration>
+							<failOnViolation>false</failOnViolation>
+                        </configuration>
                     </execution>
                 </executions>
             </plugin>
@@ -609,7 +644,7 @@
                             <consoleOutput>true</consoleOutput>
                             <includeTestSourceDirectory>true</includeTestSourceDirectory>
                             <configLocation>falcon/checkstyle.xml</configLocation>
-                            <failOnViolation>true</failOnViolation>
+							<failOnViolation>true</failOnViolation>
                             <skip>${skipCheck}</skip>
                         </configuration>
                     </execution>


Mime
View raw message