falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From peeyu...@apache.org
Subject [1/2] falcon git commit: FALCON-1627 Provider integration with Azure Data Factory pipelines
Date Fri, 19 Feb 2016 04:35:46 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 9979a1fd8 -> ad18b024f


http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/addons/adf/src/main/java/org/apache/falcon/adfservice/util/ADFJsonConstants.java
----------------------------------------------------------------------
diff --git a/addons/adf/src/main/java/org/apache/falcon/adfservice/util/ADFJsonConstants.java
b/addons/adf/src/main/java/org/apache/falcon/adfservice/util/ADFJsonConstants.java
new file mode 100644
index 0000000..9e48685
--- /dev/null
+++ b/addons/adf/src/main/java/org/apache/falcon/adfservice/util/ADFJsonConstants.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.adfservice.util;
+
+/**
+ * ADF JSON Constants in ADF request.
+ */
+public final class ADFJsonConstants {
+
+    public static final String ADF_REQUEST_ACTIVITY = "activity";
+    public static final String ADF_REQUEST_TRANSFORMATION = "transformation";
+    public static final String ADF_REQUEST_TYPE = "type";
+    public static final String ADF_REQUEST_JOBID = "jobId";
+    public static final String ADF_REQUEST_START_TIME = "dataSliceStart";
+    public static final String ADF_REQUEST_END_TIME = "dataSliceEnd";
+    public static final String ADF_REQUEST_SCHEDULER = "scheduler";
+    public static final String ADF_REQUEST_POLICY = "policy";
+    public static final String ADF_REQUEST_TIMEOUT = "timeout";
+    public static final String ADF_REQUEST_FREQUENCY = "frequency";
+    public static final String ADF_REQUEST_INTERVAL = "interval";
+    public static final String ADF_REQUEST_LINKED_SERVICES = "linkedServices";
+    public static final String ADF_REQUEST_NAME = "name";
+    public static final String ADF_REQUEST_INPUTS = "inputs";
+    public static final String ADF_REQUEST_OUTPUTS = "outputs";
+    public static final String ADF_REQUEST_TABLES = "tables";
+    public static final String ADF_REQUEST_PROPERTIES = "properties";
+    public static final String ADF_REQUEST_EXTENDED_PROPERTIES = "extendedProperties";
+    public static final String ADF_REQUEST_CLUSTER_NAME = "clusterName";
+    public static final String ADF_REQUEST_RUN_ON_BEHALF_USER = "runOnBehalf";
+    public static final String ADF_REQUEST_LOCATION = "location";
+    public static final String ADF_REQUEST_FOLDER_PATH = "folderPath";
+    public static final String ADF_REQUEST_SCRIPT = "script";
+    public static final String ADF_REQUEST_SCRIPT_PATH = "scriptPath";
+    public static final String ADF_REQUEST_LINKED_SERVICE_NAME = "linkedServiceName";
+    public static final String ADF_REQUEST_TABLE_NAME = "tableName";
+    public static final String ADF_REQUEST_TABLE_PARTITION = "partitionedBy";
+    public static final String ADF_REQUEST_LOCATION_TYPE_AZURE_BLOB = "AzureBlobLocation";
+    public static final String ADF_REQUEST_CONNECTION_STRING = "connectionString";
+    public static final String ADF_REQUEST_BLOB_ACCOUNT_NAME = "AccountName=";
+
+    public static final String ADF_STATUS_PROTOCOL = "TransportProtocolVersion";
+    public static final String ADF_STATUS_JOBID = "JobId";
+    public static final String ADF_STATUS_STATUS = "Status";
+    public static final String ADF_STATUS_PROGRESS = "Progress";
+    public static final String ADF_STATUS_LOG_URL = "LogURL";
+    public static final String ADF_STATUS_ERROR_TYPE = "ErrorType";
+    public static final String ADF_STATUS_ERROR_MESSAGE = "ErrorMessage";
+    public static final String ADF_STATUS_PROTOCOL_NAME = "2.0-preview";
+    public static final String ADF_STATUS_ERROR_TYPE_VALUE = "UserError";
+    public static final String ADF_STATUS_SUCCEEDED = "Succeeded";
+    public static final String ADF_STATUS_FAILED = "Failed";
+    public static final String ADF_STATUS_EXECUTING = "Executing";
+    public static final String ADF_STATUS_CANCELED = "Canceled";
+
+    private ADFJsonConstants() {
+    }
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/addons/adf/src/main/java/org/apache/falcon/adfservice/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/addons/adf/src/main/java/org/apache/falcon/adfservice/util/FSUtils.java b/addons/adf/src/main/java/org/apache/falcon/adfservice/util/FSUtils.java
new file mode 100644
index 0000000..58abfbf
--- /dev/null
+++ b/addons/adf/src/main/java/org/apache/falcon/adfservice/util/FSUtils.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.adfservice.util;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.falcon.FalconException;
+import org.apache.falcon.hadoop.HadoopClientFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.URISyntaxException;
+
+/**
+ * Utility for file operations.
+ */
+public final class FSUtils {
+    private static final Logger LOG = LoggerFactory.getLogger(FSUtils.class);
+    private FSUtils() {
+    }
+
+    public static String readHDFSFile(final String filePath, final String fileName)
+        throws URISyntaxException, FalconException {
+        BufferedReader br = null;
+        try {
+            Path path = new Path(filePath, fileName);
+            FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(path.toUri());
+            br = new BufferedReader(new InputStreamReader(fs.open(path)));
+            StringBuilder fileContent = new StringBuilder();
+            String line;
+            while (true) {
+                line = br.readLine();
+                if (line == null) {
+                    break;
+                }
+                fileContent.append(line);
+            }
+            return fileContent.toString();
+        } catch (IOException e) {
+            throw new FalconException("Error reading file from hdfs: " + filePath + fileName
+ ": " + e.toString(), e);
+        } finally {
+            IOUtils.closeQuietly(br);
+        }
+    }
+
+    public static String createFile(final Path path,
+                                    final String content) throws FalconException {
+        OutputStream out = null;
+        try {
+            FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(path.toUri());
+            out = fs.create(path);
+            out.write(content.getBytes());
+        } catch (IOException e) {
+            throw new FalconException("Error preparing script file: " + path, e);
+        } finally {
+            IOUtils.closeQuietly(out);
+        }
+        return path.toString();
+    }
+
+    public static void createDir(final Path dirPath) throws FalconException {
+        FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(dirPath.toUri());
+        try {
+            if (!fs.exists(dirPath)) {
+                LOG.info("Creating directory: {}", dirPath);
+                HadoopClientFactory.mkdirsWithDefaultPerms(fs, dirPath);
+            }
+        } catch (IOException e) {
+            throw new FalconException("Error creating directory: " + dirPath, e);
+        }
+    }
+
+    public static void removeDir(final Path dirPath) throws FalconException {
+        FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(dirPath.toUri());
+        try {
+            fs.delete(dirPath, true);
+        } catch (IOException e) {
+            throw new FalconException("Error creating directory: " + dirPath, e);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/addons/hivedr/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hivedr/pom.xml b/addons/hivedr/pom.xml
index a4d6be8..37dc5c9 100644
--- a/addons/hivedr/pom.xml
+++ b/addons/hivedr/pom.xml
@@ -64,7 +64,12 @@
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-jdbc</artifactId>
             <version>${hive.version}</version>
-            <classifier>standalone</classifier>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.httpcomponents</groupId>
+                    <artifactId>httpclient</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.testng</groupId>

http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/common/src/main/resources/startup.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties
index 3f1ed03..2497cce 100644
--- a/common/src/main/resources/startup.properties
+++ b/common/src/main/resources/startup.properties
@@ -42,7 +42,8 @@
                         org.apache.falcon.metadata.MetadataMappingService,\
                         org.apache.falcon.service.LogCleanupService,\
                         org.apache.falcon.service.GroupsService,\
-                        org.apache.falcon.service.ProxyUserService
+                        org.apache.falcon.service.ProxyUserService,\
+                        org.apache.falcon.adfservice.ADFProviderService
 ## If you wish to use Falcon native scheduler add the commented out services below to application.services
##
 #                        org.apache.falcon.notification.service.impl.JobCompletionService,\
 #                        org.apache.falcon.notification.service.impl.SchedulerService,\
@@ -234,6 +235,33 @@ it.workflow.execution.listeners=org.apache.falcon.catalog.CatalogPartitionHandle
 
 ######### Authorization Properties #########
 
+######### ADF Configurations start #########
+
+# A String object that represents the namespace
+*.microsoft.windowsazure.services.servicebus.namespace=
+
+# Request and status queues on the namespace
+*.microsoft.windowsazure.services.servicebus.requestqueuename=
+*.microsoft.windowsazure.services.servicebus.statusqueuename=
+
+# A String object that contains the SAS key name
+*.microsoft.windowsazure.services.servicebus.sasKeyName=
+
+# A String object that contains the SAS key
+*.microsoft.windowsazure.services.servicebus.sasKey=
+
+# A String object containing the base URI that is added to your Service Bus namespace to
form the URI to connect
+# to the Service Bus service. To access the default public Azure service, pass ".servicebus.windows.net"
+*.microsoft.windowsazure.services.servicebus.serviceBusRootUri=
+
+# Service bus polling frequency
+*.microsoft.windowsazure.services.servicebus.polling.frequency=
+
+# Super user
+*.microsoft.windowsazure.services.servicebus.superuser=
+
+######### ADF Configurations end ###########
+
 ######### SMTP Properties ########
 
 # Setting SMTP hostname

http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fbf5913..27d2fc2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -402,6 +402,12 @@
                 <module>addons/hivedr</module>
             </modules>
         </profile>
+        <profile>
+            <id>adf</id>
+            <modules>
+                <module>addons/adf</module>
+            </modules>
+        </profile>
     </profiles>
 
     <modules>
@@ -539,13 +545,13 @@
             <dependency>
                 <groupId>org.codehaus.jackson</groupId>
                 <artifactId>jackson-core-asl</artifactId>
-                <version>1.5.2</version>
+                <version>1.9.2</version>
             </dependency>
 
             <dependency>
                 <groupId>org.codehaus.jackson</groupId>
                 <artifactId>jackson-mapper-asl</artifactId>
-                <version>1.5.2</version>
+                <version>1.9.2</version>
             </dependency>
 
             <dependency>
@@ -965,6 +971,10 @@
                         <groupId>org.apache.derby</groupId>
                         <artifactId>derby</artifactId>
                     </exclusion>
+                    <exclusion> <!-- conflict with hadoop-auth -->
+                        <groupId>org.apache.httpcomponents</groupId>
+                        <artifactId>httpclient</artifactId>
+                    </exclusion>
                 </exclusions>
             </dependency>
 
@@ -977,6 +987,10 @@
                         <groupId>org.apache.hadoop</groupId>
                         <artifactId>hadoop-client</artifactId>
                     </exclusion>
+                    <exclusion> <!-- conflict with hadoop-auth -->
+                        <groupId>org.apache.httpcomponents</groupId>
+                        <artifactId>httpclient</artifactId>
+                    </exclusion>
                 </exclusions>
             </dependency>
 
@@ -1034,6 +1048,18 @@
             </dependency>
 
             <dependency>
+                <groupId>org.apache.httpcomponents</groupId>
+                <artifactId>httpclient</artifactId>
+                <version>4.3.3</version>
+            </dependency>
+
+            <dependency>
+                <groupId>org.apache.httpcomponents</groupId>
+                <artifactId>httpcore</artifactId>
+                <version>4.3.2</version>
+            </dependency>
+
+            <dependency>
                 <groupId>com.tinkerpop.blueprints</groupId>
                 <artifactId>blueprints-core</artifactId>
                 <version>2.5.0</version>

http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/src/conf/startup.properties
----------------------------------------------------------------------
diff --git a/src/conf/startup.properties b/src/conf/startup.properties
index b1a340a..faaedfa 100644
--- a/src/conf/startup.properties
+++ b/src/conf/startup.properties
@@ -50,7 +50,8 @@
                         org.apache.falcon.metadata.MetadataMappingService,\
                         org.apache.falcon.service.LogCleanupService,\
                         org.apache.falcon.service.GroupsService,\
-                        org.apache.falcon.service.ProxyUserService
+                        org.apache.falcon.service.ProxyUserService,\
+                        org.apache.falcon.adfservice.ADFProviderService
 ## If you wish to use Falcon native scheduler uncomment out below  application services and
comment out above application services ##
 #*.application.services=org.apache.falcon.security.AuthenticationInitializationService,\
 #                        org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
@@ -251,6 +252,33 @@ prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\
 
 ######### Authorization Properties #########
 
+######### ADF Configurations start #########
+
+# A String object that represents the namespace
+*.microsoft.windowsazure.services.servicebus.namespace=
+
+# Request and status queues on the namespace
+*.microsoft.windowsazure.services.servicebus.requestqueuename=
+*.microsoft.windowsazure.services.servicebus.statusqueuename=
+
+# A String object that contains the SAS key name
+*.microsoft.windowsazure.services.servicebus.sasKeyName=
+
+# A String object that contains the SAS key
+*.microsoft.windowsazure.services.servicebus.sasKey=
+
+# A String object containing the base URI that is added to your Service Bus namespace to
form the URI to connect
+# to the Service Bus service. To access the default public Azure service, pass ".servicebus.windows.net"
+*.microsoft.windowsazure.services.servicebus.serviceBusRootUri=
+
+# Service bus polling frequency
+*.microsoft.windowsazure.services.servicebus.polling.frequency=
+
+# Super user
+*.microsoft.windowsazure.services.servicebus.superuser=
+
+######### ADF Configurations end ###########
+
 ######### SMTP Properties ########
 
 # Setting SMTP hostname

http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 3996966..06aa897 100644
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -71,6 +71,10 @@
                     <artifactId>hadoop-client</artifactId>
                     <scope>test</scope>
                 </dependency>
+                <dependency>
+                    <groupId>org.apache.httpcomponents</groupId>
+                    <artifactId>httpclient</artifactId>
+                </dependency>
             </dependencies>
         </profile>
         <profile>
@@ -83,6 +87,16 @@
                 </dependency>
             </dependencies>
         </profile>
+        <profile>
+            <id>adf</id>
+            <dependencies>
+                <dependency>
+                    <groupId>org.apache.falcon</groupId>
+                    <artifactId>falcon-adf</artifactId>
+                    <version>${project.version}</version>
+                </dependency>
+            </dependencies>
+        </profile>
     </profiles>
 
     <dependencies>


Mime
View raw message