ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [1/5] ambari git commit: AMBARI-11034. Implement HdfsResouces and it's usage (aonishuk)
Date Fri, 08 May 2015 20:38:50 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 92da024a5 -> 6e8dce443


http://git-wip-us.apache.org/repos/asf/ambari/blob/6e8dce44/ambari-server/src/test/python/stacks/2.2/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/default.json b/ambari-server/src/test/python/stacks/2.2/configs/default.json
index c5a6ae7..8188928 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/default.json
@@ -185,7 +185,18 @@
             "ignore_groupsusers_create": "false",
             "smokeuser": "ambari-qa",
             "kerberos_domain": "EXAMPLE.COM",
-            "user_group": "hadoop"
+            "user_group": "hadoop",
+	        "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", 
+	        "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version
}}/mapreduce/", 
+	        "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", 
+	        "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/",

+	        "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/", 
+	        "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/",

+	        "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz", 
+	        "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/", 
+	        "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", 
+	        "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/",

+	        "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz"
         },
         "ranger-knox-plugin-properties": {
             "POLICY_MGR_URL": "{{policymgr_mgr_url}}", 

http://git-wip-us.apache.org/repos/asf/ambari/blob/6e8dce44/ambari-server/src/test/python/stacks/2.2/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/secured.json b/ambari-server/src/test/python/stacks/2.2/configs/secured.json
index 5bd8814..e224ebc 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/secured.json
@@ -173,7 +173,18 @@
             "user_group": "hadoop",
             "smokeuser_keytab": "/etc/security/keytabs/smokeuser.headless.keytab",
             "smokeuser_principal_name": "ambari-qa@EXAMPLE.COM",
-            "kinit_path_local": "/usr/bin"
+            "kinit_path_local": "/usr/bin",
+	        "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz", 
+	        "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version
}}/mapreduce/", 
+	        "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz", 
+	        "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/",

+	        "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/", 
+	        "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/",

+	        "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz", 
+	        "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/", 
+	        "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz", 
+	        "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/",

+	        "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz"
         },
         "webhcat-site": {
             "templeton.jar": "/usr/hdp/current/hive-webhcat/share/webhcat/svr/lib/hive-webhcat-*.jar",

http://git-wip-us.apache.org/repos/asf/ambari/blob/6e8dce44/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
b/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
index 0d943c4..665119f 100644
--- a/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_service_check.py
@@ -33,35 +33,42 @@ class TestMahoutClient(RMFTestCase):
                        target = RMFTestCase.TARGET_COMMON_SERVICES
     )
 
-    self.assertResourceCalled('ExecuteHadoop', 'fs -rm -r -f /user/ambari-qa/mahoutsmokeoutput
/user/ambari-qa/mahoutsmokeinput',
-                              security_enabled = False,
-                              keytab = UnknownConfigurationMock(),
-                              conf_dir = '/usr/hdp/current/hadoop-client/conf',
-                              try_sleep = 5,
-                              kinit_path_local = '/usr/bin/kinit',
-                              tries = 3,
-                              user = 'ambari-qa',
-                              bin_dir = '/usr/hdp/current/hadoop-client/bin',
-                              principal = UnknownConfigurationMock(),
-                              )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -mkdir /user/ambari-qa/mahoutsmokeinput',
-                              try_sleep = 5,
-                              tries = 3,
-                              bin_dir = '/usr/hdp/current/hadoop-client/bin',
-                              user = 'ambari-qa',
-                              conf_dir = '/usr/hdp/current/hadoop-client/conf',
-                              )
     self.assertResourceCalled('File', '/tmp/sample-mahout-test.txt',
-                              content = 'Test text which will be converted to sequence file.',
-                              mode = 0755,
-                              )
-    self.assertResourceCalled('ExecuteHadoop', 'fs -put /tmp/sample-mahout-test.txt /user/ambari-qa/mahoutsmokeinput/',
-                              try_sleep = 5,
-                              tries = 3,
-                              bin_dir = '/usr/hdp/current/hadoop-client/bin',
-                              user = 'ambari-qa',
-                              conf_dir = '/usr/hdp/current/hadoop-client/conf',
-                              )
+        content = 'Test text which will be converted to sequence file.',
+        mode = 0755,
+    )
+    self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mahoutsmokeinput',
+        security_enabled = False,
+        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+        keytab = UnknownConfigurationMock(),
+        kinit_path_local = '/usr/bin/kinit',
+        user = 'hdfs',
+        owner = 'ambari-qa',
+        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+        type = 'directory',
+        action = ['create_on_execute'],
+    )
+    self.assertResourceCalled('HdfsResource', '/user/ambari-qa/mahoutsmokeinput/sample-mahout-test.txt',
+        security_enabled = False,
+        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+        keytab = UnknownConfigurationMock(),
+        kinit_path_local = '/usr/bin/kinit',
+        source = '/tmp/sample-mahout-test.txt',
+        user = 'hdfs',
+        owner = 'ambari-qa',
+        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+        type = 'file',
+        action = ['create_on_execute'],
+    )
+    self.assertResourceCalled('HdfsResource', None,
+        security_enabled = False,
+        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+        keytab = UnknownConfigurationMock(),
+        kinit_path_local = '/usr/bin/kinit',
+        user = 'hdfs',
+        action = ['execute'],
+        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+    )
     self.assertResourceCalled('Execute', 'mahout seqdirectory --input /user/ambari-qa/mahoutsmokeinput/'
                                          'sample-mahout-test.txt --output /user/ambari-qa/mahoutsmokeoutput/
'
                                          '--charset utf-8',

http://git-wip-us.apache.org/repos/asf/ambari/blob/6e8dce44/contrib/fast-hdfs-resource/dependency-reduced-pom.xml
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/dependency-reduced-pom.xml b/contrib/fast-hdfs-resource/dependency-reduced-pom.xml
new file mode 100644
index 0000000..c252f2e
--- /dev/null
+++ b/contrib/fast-hdfs-resource/dependency-reduced-pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.ambari</groupId>
+  <artifactId>fast-hdfs-resource</artifactId>
+  <name>fast-hdfs-resource</name>
+  <version>0.0.1-SNAPSHOT</version>
+  <url>http://maven.apache.org</url>
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>2.3</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <transformers>
+                <transformer>
+                  <mainClass>org.apache.ambari.fast_hdfs_resource.Runner</mainClass>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+  <repositories>
+    <repository>
+      <id>hdp.internal</id>
+      <url>http://repo1.maven.org/maven2</url>
+    </repository>
+  </repositories>
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+</project>
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/6e8dce44/contrib/fast-hdfs-resource/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/pom.xml b/contrib/fast-hdfs-resource/pom.xml
new file mode 100644
index 0000000..6a92841
--- /dev/null
+++ b/contrib/fast-hdfs-resource/pom.xml
@@ -0,0 +1,86 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.apache.ambari</groupId>
+  <artifactId>fast-hdfs-resource</artifactId>
+  <version>0.0.1-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <name>fast-hdfs-resource</name>
+  <url>http://maven.apache.org</url>
+  <repositories>
+    <repository>
+      <id>hdp.internal</id>
+      <url>http://repo1.maven.org/maven2</url>
+    </repository>
+  </repositories>
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-tools</artifactId>
+      <version>1.2.1</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-core</artifactId>
+      <version>1.2.1</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.code.gson</groupId>
+      <artifactId>gson</artifactId>
+      <version>2.2.2</version>
+    </dependency>
+  </dependencies>
+
+
+  <!-- Create executable jar with the application entry point -->
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>2.3</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <transformers>
+                <transformer
+                  implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                  <mainClass>org.apache.ambari.fast_hdfs_resource.Runner
+                  </mainClass>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

http://git-wip-us.apache.org/repos/asf/ambari/blob/6e8dce44/contrib/fast-hdfs-resource/resources/example.json
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/resources/example.json b/contrib/fast-hdfs-resource/resources/example.json
new file mode 100644
index 0000000..605deb9
--- /dev/null
+++ b/contrib/fast-hdfs-resource/resources/example.json
@@ -0,0 +1,57 @@
+[
+{
+	"target":"/tmp/some999",
+	"type":"directory",
+	"action":"delete"
+},
+{
+	"target":"/tmp/some999/more/dirs/for/recursive/tests",
+	"type":"directory",
+	"action":"create"
+},
+{
+	"target":"/tmp/some999/more/dirs/for/recursive/tests/file_empty.txt",
+	"type":"file",
+	"action":"create"
+},
+{
+	"target":"/tmp/some999",
+	"type":"directory",
+	"action":"create",
+	"owner":"oozie"
+},
+{
+	"target":"/tmp/some999",
+	"type":"directory",
+	"action":"create",
+	"group":"hive"
+},
+{
+	"target":"/tmp/some999",
+	"type":"directory",
+	"action":"create",
+	"mode":"777"
+},
+{
+	"target":"/tmp/some999/more/dirs",
+	"type":"directory",
+	"action":"create",
+	"owner":"yarn",
+	"group":"mapred",
+	"recursiveChown":true,
+	"mode":"757",
+	"recursiveChmod":true
+},
+{
+	"source":"/tmp/my.txt",
+	"target":"/tmp/some999/my_file.txt",
+	"type":"file",
+	"action":"create"
+},
+{
+	"source":"/tmp/a",
+	"target":"/tmp/some999/a_dir",
+	"type":"directory",
+	"action":"create"
+}
+]
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/6e8dce44/contrib/fast-hdfs-resource/resources/test_perfomance.sh
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/resources/test_perfomance.sh b/contrib/fast-hdfs-resource/resources/test_perfomance.sh
new file mode 100644
index 0000000..40339e3
--- /dev/null
+++ b/contrib/fast-hdfs-resource/resources/test_perfomance.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Delete 2 dirs
+sudo -u hdfs hadoop fs -rm -r /tmp/some999
+sudo -u hdfs hadoop fs -rm -r /tmp/some888
+# Create
+sudo -u hdfs hadoop fs -mkdir -p /tmp/some999/more/dirs/for/recursive/tests
+# Create + permissions + owner
+sudo -u hdfs hadoop fs -mkdir -p /tmp/some888/more/dirs/for/recursive/tests
+sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/more/dirs/for/recursive/tests
+sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/more/dirs/for/recursive/tests
+# Empty dirs with permissions/owners to last dir"
+sudo -u hdfs hadoop fs -mkdir -p /tmp/some888/and_more/and_dirs/_andfor/recursive/tests
+sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/and_more/and_dirs/_andfor/recursive/tests
+sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/and_more/and_dirs/_andfor/recursive/tests
+# Empty dirs with permissions/owners to last file
+sudo -u hdfs hadoop fs -touchz /tmp/some888/file.txt
+sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/file.txt
+sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/file.txt
+# Empty dirs with permissions/owners to last file
+sudo -u hdfs hadoop fs -touchz /tmp/some888/and_more/and_dirs/file2.txt
+sudo -u hdfs hadoop fs -chown hadoop:hadoop /tmp/some888/and_more/and_dirs/file2.txt
+sudo -u hdfs hadoop fs -chmod 777 /tmp/some888/and_more/and_dirs/file2.txt
+# Recursive permissions
+sudo -u hdfs hadoop fs -chmod -R 700 /tmp/some888
+sudo -u hdfs hadoop fs -chown -R hive:hive /tmp/some999
+
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/6e8dce44/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java
b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java
new file mode 100644
index 0000000..21750e1
--- /dev/null
+++ b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Resource.java
@@ -0,0 +1,295 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.fast_hdfs_resource;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.File;
+import java.lang.System;
+import java.util.ArrayList;
+import java.util.HashSet;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * Used to: 1) copy files/directories from localFS to hadoopFs 2) create empty
+ * files/directories in hadoopFs
+ */
+public class Resource {
+  private String source;
+  private String target;
+  private String type;
+  private String action;
+  private String owner;
+  private String group;
+  private String mode;
+  private boolean recursiveChown;
+  private boolean recursiveChmod;
+  private boolean changePermissionforParents;
+
+  public String getSource() {
+    return source;
+  }
+
+  public void setSource(String source) {
+    this.source = source;
+  }
+
+  public String getTarget() {
+    return target;
+  }
+
+  public void setTarget(String target) {
+    this.target = target;
+  }
+
+  public String getType() {
+    return type;
+  }
+
+  public void setType(String type) {
+    this.type = type;
+  }
+
+  public String getAction() {
+    return action;
+  }
+
+  public void setAction(String action) {
+    this.action = action;
+  }
+
+  public String getOwner() {
+    return owner;
+  }
+
+  public void setOwner(String owner) {
+    this.owner = owner;
+  }
+
+  public String getGroup() {
+    return group;
+  }
+
+  public void setGroup(String group) {
+    this.group = group;
+  }
+
+  public String getMode() {
+    return mode;
+  }
+
+  public void setMode(String mode) {
+    this.mode = mode;
+  }
+
+  public boolean isRecursiveChown() {
+    return recursiveChown;
+  }
+
+  public void setRecursiveChown(boolean recursiveChown) {
+    this.recursiveChown = recursiveChown;
+  }
+
+  public boolean isRecursiveChmod() {
+    return recursiveChmod;
+  }
+
+  public void setRecursiveChmod(boolean recursiveChmod) {
+    this.recursiveChmod = recursiveChmod;
+  }
+  
+  public boolean isChangePermissionOnParents() {
+    return changePermissionforParents;
+  }
+
+  public void setChangePermissionOnParents(boolean changePermissionforParents) {
+    this.changePermissionforParents = changePermissionforParents;
+  }
+
+  
+  
+  
+  @Override
+  public String toString() {
+    return "Resource [source=" + source + ", target=" + target + ", type="
+        + type + ", action=" + action + ", owner=" + owner + ", group=" + group
+        + ", mode=" + mode + ", recursiveChown=" + recursiveChown
+        + ", recursiveChmod=" + recursiveChmod
+        + ", changePermissionforParents=" + changePermissionforParents + "]";
+  }
+
+  /*
+   * Check if parameters are correctly set
+   */
+  public static void checkResourceParameters(Resource resource,
+      FileSystem dfs) throws IllegalArgumentException, IOException {
+
+    ArrayList<String> actionsAvailable = new ArrayList<String>();
+    actionsAvailable.add("create");
+    actionsAvailable.add("delete");
+    ArrayList<String> typesAvailable = new ArrayList<String>();
+    typesAvailable.add("file");
+    typesAvailable.add("directory");
+
+    if (resource.getTarget() == null)
+      throw new IllegalArgumentException(
+          "Path to resource in HadoopFs must be filled.");
+
+    if (resource.getAction() == null
+        || !actionsAvailable.contains(resource.getAction()))
+      throw new IllegalArgumentException("Action is not supported.");
+
+    if (resource.getType() == null
+        || !typesAvailable.contains(resource.getType()))
+      throw new IllegalArgumentException("Type is not supported.");
+
+    // Check consistency for ("type":"file" == file in hadoop)
+    if (dfs.isFile(new Path(resource.getTarget()))
+        && !"file".equals(resource.getType()))
+      throw new IllegalArgumentException(
+          "Cannot create a directory " + resource.getTarget() +
+              " because file is present on the given path.");
+    // Check consistency for ("type":"directory" == directory in hadoop)
+    else if (dfs.isDirectory(new Path(resource.getTarget()))
+        && !"directory".equals(resource.getType()))
+      throw new IllegalArgumentException(
+          "Cannot create a file " + resource.getTarget() +
+              " because directory is present on the given path.");
+    
+    if(resource.getSource() != null) {
+      File source = new File(resource.getSource());
+      if(source.isFile()
+          && !"file".equals(resource.getType()))
+        throw new IllegalArgumentException(
+            "Cannot create a directory " + resource.getTarget() +
+                " because source " + resource.getSource() + "is a file");
+      else if(source.isDirectory()
+          && !"directory".equals(resource.getType()))
+        throw new IllegalArgumentException(
+            "Cannot create a file " + resource.getTarget() +
+                " because source " + resource.getSource() + "is a directory");      
+    }
+  }
+
+  /*
+   * Create/copy resource - {type}
+   */
+  public static void createResource(Resource resource,
+      FileSystem dfs, Path pathHadoop) throws IOException {
+
+    boolean isCreate = (resource.getSource() == null) ? true : false;
+
+    if (isCreate && resource.getType().equals("directory")) {
+      dfs.mkdirs(pathHadoop); // empty dir(s)
+    } else if (isCreate && resource.getType().equals("file")) {
+      dfs.createNewFile(pathHadoop); // empty file
+    } else {
+      dfs.copyFromLocalFile(new Path(resource.getSource()), pathHadoop);// copy
+    }
+  }
+
+  /*
+   * Set permissions on resource - {mode}
+   */
+  public static void setMode(Resource resource,
+      FileSystem dfs, Path pathHadoop) throws IOException {
+
+    if (resource.getMode() != null) {
+      FsPermission permission = new FsPermission(resource.getMode());
+      dfs.setPermission(pathHadoop, permission);
+
+      // Recursive
+      
+        // Get the list of sub-directories and files
+        HashSet<String> resultSet = new HashSet<String>();
+        
+        if (resource.isRecursiveChmod())
+          resource.fillDirectoryList(dfs, resource.getTarget(), resultSet);
+        
+        if(resource.isChangePermissionOnParents())
+          resource.fillInParentDirectories(dfs, resource.getTarget(), resultSet);
+
+        for (String path : resultSet) {
+          dfs.setPermission(new Path(path), permission);
+        }
+
+    }
+  }
+
+  /*
+   * Set owner on resource - {owner}
+   */
+  public static void setOwner(Resource resource, FileSystem dfs,
+      Path pathHadoop) throws IOException {
+
+    if (!(resource.getOwner() == null && resource.getGroup() == null)) {
+      dfs.setOwner(pathHadoop, resource.getOwner(), resource.getGroup());
+
+      // Get the list of sub-directories and files
+      HashSet<String> resultSet = new HashSet<String>();
+      if (resource.isRecursiveChown())
+        resource.fillDirectoryList(dfs, resource.getTarget(), resultSet);
+      if(resource.isChangePermissionOnParents())
+        resource.fillInParentDirectories(dfs, resource.getTarget(), resultSet);
+
+      for (String path : resultSet) {
+        dfs.setOwner(new Path(path), resource.getOwner(), resource.getGroup());
+      }
+    }
+  }
+  
+  public void fillInParentDirectories(FileSystem dfs, String path, HashSet<String>
resultSet) throws IOException {
+    Path filePath = new Path(path);
+      
+    while(true) {
+      filePath = filePath.getParent();
+      
+      // if(filePath.isRoot()) {
+      if(filePath.getParent() == null) {
+        break;
+      }
+      resultSet.add(filePath.toString());
+    }
+  }
+
+  /*
+   * List all files and sub-directories recursively
+   */
+  public void fillDirectoryList(FileSystem dfs, String path,
+      HashSet<String> resultSet) throws IOException {
+
+    FileStatus[] fileStatus = dfs.listStatus(new Path(path));
+    if (fileStatus != null) {
+      // Go through all resources in directory
+      for (FileStatus fs : fileStatus) {
+        String pathToResource = path + "/" + fs.getPath().getName();
+
+        resultSet.add(pathToResource);
+
+        if (fs.isDir()) {
+          // recursive
+          fillDirectoryList(dfs, pathToResource, resultSet);
+        }
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/6e8dce44/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
----------------------------------------------------------------------
diff --git a/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
new file mode 100644
index 0000000..291a2d9
--- /dev/null
+++ b/contrib/fast-hdfs-resource/src/main/java/org/apache/ambari/fast_hdfs_resource/Runner.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.fast_hdfs_resource;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+
+import com.google.gson.Gson;
+
+public class Runner {
+  public static void main(String[] args)
+      throws IOException, URISyntaxException {
+    // 1 - Check arguments
+    if (args.length != 1) {
+      System.err.println("Incorrect number of arguments. Please provide:\n"
+          + "1) Path to json file\n"
+          + "Exiting...");
+      System.exit(1);
+    }
+
+    // 2 - Check if json-file exists
+    final String jsonFilePath = args[0];
+    File file = new File(jsonFilePath);
+
+    if (!file.isFile()) {
+      System.err
+          .println("File " + jsonFilePath + " doesn't exist.\nExiting...");
+      System.exit(1);
+    }
+
+    Gson gson = new Gson();
+    Resource[] resources = null;
+    FileSystem dfs = null;
+
+    try {
+      Configuration conf = new Configuration();
+      dfs = FileSystem.get(conf);
+
+      // 3 - Load data from JSON
+      resources = (Resource[]) gson.fromJson(new FileReader(jsonFilePath),
+          Resource[].class);
+
+      // 4 - Connect to HDFS
+      System.out.println("Using filesystem uri: " + FileSystem.getDefaultUri(conf).toString());
+      dfs.initialize(FileSystem.getDefaultUri(conf), conf);
+      
+      for (Resource resource : resources) {
+        System.out.println("Creating: " + resource);
+
+        Resource.checkResourceParameters(resource, dfs);
+
+        Path pathHadoop = new Path(resource.getTarget());
+        if (resource.getAction().equals("create")) {
+          // 5 - Create
+          Resource.createResource(resource, dfs, pathHadoop);
+          Resource.setMode(resource, dfs, pathHadoop);
+          Resource.setOwner(resource, dfs, pathHadoop);
+        } else if (resource.getAction().equals("delete")) {
+          // 6 - Delete
+          dfs.delete(pathHadoop, true);
+        }
+      }
+
+    } finally {
+      dfs.close();
+    }
+
+    System.out.println("All resources created.");
+  }
+
+}


Mime
View raw message