bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject [2/2] git commit: BIGTOP-1188. Smoke tests for HttpFs
Date Thu, 27 Feb 2014 19:46:38 GMT
BIGTOP-1188. Smoke tests for HttpFs


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/69f7c942
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/69f7c942
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/69f7c942

Branch: refs/heads/master
Commit: 69f7c9426c4a80dde7c9b1509120cd8600c65976
Parents: d4504ae
Author: Adam Kawa <kawa.adam@gmail.com>
Authored: Tue Jan 21 09:37:55 2014 +0000
Committer: vagrant <vagrant@ubuntu-server-12042-x64-vbox4210.(none)>
Committed: Thu Feb 27 19:26:09 2014 +0000

----------------------------------------------------------------------
 bigtop-tests/test-artifacts/httpfs/pom.xml      |  35 +++++
 .../bigtop/itest/httpfs/TestHttpFs.groovy       | 153 +++++++++++++++++++
 .../main/resources/text-files/helloworld.txt    |   1 +
 bigtop-tests/test-artifacts/pom.xml             |   1 +
 .../test-execution/smokes/httpfs/pom.xml        |  97 ++++++++++++
 bigtop-tests/test-execution/smokes/pom.xml      |   9 +-
 6 files changed, 295 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/httpfs/pom.xml b/bigtop-tests/test-artifacts/httpfs/pom.xml
new file mode 100644
index 0000000..aafa8f3
--- /dev/null
+++ b/bigtop-tests/test-artifacts/httpfs/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.bigtop.itest</groupId>
+    <artifactId>bigtop-smokes</artifactId>
+    <version>0.8.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <groupId>org.apache.bigtop.itest</groupId>
+  <artifactId>httpfs-smoke</artifactId>
+  <version>0.8.0-SNAPSHOT</version>
+  <name>httpfssmokes</name>
+
+</project>

http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy
b/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy
new file mode 100644
index 0000000..0aa9a45
--- /dev/null
+++ b/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bigtop.itest.hadoop.hdfs
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.apache.bigtop.itest.shell.Shell;
+import org.apache.bigtop.itest.TestUtils
+import org.apache.commons.logging.Log
+import org.apache.commons.logging.LogFactory
+
+public class TestHttpFs {
+    private static Log LOG = LogFactory.getLog(TestHttpFs.class)
+ 
+    private static final String USERNAME = System.getProperty("user.name");
+    private static final String HTTPFS_PROXY = System.getenv('HTTPFS_PROXY');
+    static {
+        assertNotNull("HTTPFS_PROXY has to be set to run this test",
+            HTTPFS_PROXY);
+    }
+  
+    private static final String HTTPFS_PREFIX = "http://$HTTPFS_PROXY/webhdfs/v1";
+    private static final String HTTPFS_SUCCESS = "{\"boolean\":true}";
+    
+    private static final String DATA_DIR = System.getProperty("data.dir", "text-files");
+        
+    private static String testHttpFsFolder =  "/tmp/httpfssmoke-" + (new Date().getTime());
+    private static String testHttpFsFolderRenamed = "$testHttpFsFolder-renamed";
+    
+    private static Shell shHDFS = new Shell("/bin/bash", USERNAME);
+    private static Shell sh = new Shell("/bin/bash");
+    
+
+    @BeforeClass
+    public static void setUp() {
+    }
+
+    @AfterClass
+    public static void tearDown() {
+        // clean up of existing folders
+        shHDFS.exec("hadoop fs -test -e $testHttpFsFolder");
+        if (shHDFS.getRet() == 0) {
+            shHDFS.exec("hadoop fs -rmr -skipTrash $testHttpFsFolder");
+            assertTrue("Deletion of previous testHttpFsFolder from HDFS failed",
+                shHDFS.getRet() == 0);
+        }
+        shHDFS.exec("hadoop fs -test -e $testHttpFsFolderRenamed");
+        if (shHDFS.getRet() == 0) {
+            shHDFS.exec("hadoop fs -rmr -skipTrash $testHttpFsFolderRenamed");
+            assertTrue("Deletion of previous testHttpFsFolderRenamed from HDFS failed",
+                shHDFS.getRet() == 0);
+        }
+    }
+
+    public void assertValueExists(List<String> values, String expected) {
+        boolean exists = false;
+        for (String value: values) {
+            if (expected.startsWith(value)) {
+                exists = true;
+            }
+        }
+        assertTrue(expected + " NOT found!", exists == true);
+    }
+    
+    private void createDir(String dirname) {
+        sh.exec("curl -i -X PUT '$HTTPFS_PREFIX$dirname?user.name=$USERNAME&op=MKDIRS'");
+    }
+
+    @Test
+    public void testCreateDir() {
+        createDir(testHttpFsFolder)
+        assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+        assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+    }
+
+    @Test
+    public void testRenameDir() { 
+        createDir(testHttpFsFolder);
+        assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+        assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+        sh.exec("curl -i -X PUT '$HTTPFS_PREFIX$testHttpFsFolder?user.name=$USERNAME&op=RENAME&destination=$testHttpFsFolderRenamed'");
+        assertTrue("curl command to rename a dir failed", shHDFS.getRet() == 0);
+        assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+    }
+
+    @Test
+    public void testDeleteDir() {
+        createDir(testHttpFsFolder);
+        assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+        assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+        sh.exec("curl -i -X DELETE '$HTTPFS_PREFIX$testHttpFsFolder?user.name=$USERNAME&op=DELETE'");
+        assertTrue("curl command to delete a dir failed", shHDFS.getRet() == 0);
+        assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+    }
+    
+    @Test
+    public void testStatusDir() { 
+        createDir(testHttpFsFolder);
+        assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+        assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+        sh.exec("curl -i '$HTTPFS_PREFIX$testHttpFsFolder?user.name=$USERNAME&op=GETFILESTATUS'");
+        assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+        assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+        assertValueExists(sh.getOut(), "DIRECTORY");
+    }
+
+    @Test
+    public void testCreateFile() {
+        String filename = "helloworld.txt";
+        String filenameContent = 'Hello World!';
+        
+        createDir(testHttpFsFolder);
+        assertTrue("curl command to create a dir failed", sh.getRet() == 0);
+        sh.exec("curl -i -X PUT '$HTTPFS_PREFIX$testHttpFsFolder/$filename?user.name=$USERNAME&op=CREATE'");
+        assertTrue("curl command to create a file failed", shHDFS.getRet() == 0);
+        String datanodeLocation = null;
+        sh.getOut().each {
+            if (it.startsWith("Location:")) {
+                datanodeLocation = it.split(' ')[1];
+                return true;
+            }
+        }
+        LOG.debug("Datanode location: $datanodeLocation");
+        assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+        sh.exec("curl -i -T $DATA_DIR/$filename '$datanodeLocation' --header 'Content-Type:application/octet-stream'");
+        assertTrue("curl command to create a file failed", shHDFS.getRet() == 0);
+        assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+        sh.exec("curl -i -L '$HTTPFS_PREFIX$testHttpFsFolder/$filename?user.name=$USERNAME&op=OPEN'");
+        assertTrue("curl command to create a file failed", shHDFS.getRet() == 0);
+        assertValueExists(sh.getOut(), HTTPFS_SUCCESS);
+        assertValueExists(sh.getOut(), filenameContent);
+    }
+}
+

http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt
b/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt
new file mode 100644
index 0000000..980a0d5
--- /dev/null
+++ b/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt
@@ -0,0 +1 @@
+Hello World!

http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/pom.xml b/bigtop-tests/test-artifacts/pom.xml
index e3963cb..a7911e8 100644
--- a/bigtop-tests/test-artifacts/pom.xml
+++ b/bigtop-tests/test-artifacts/pom.xml
@@ -35,6 +35,7 @@
   <modules>
     <module>hadoop</module>
     <module>package</module>
+    <module>httpfs</module>
     <module>oozie</module>
     <module>hive</module>
     <module>hbase</module>

http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-execution/smokes/httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-execution/smokes/httpfs/pom.xml b/bigtop-tests/test-execution/smokes/httpfs/pom.xml
new file mode 100644
index 0000000..74696e2
--- /dev/null
+++ b/bigtop-tests/test-execution/smokes/httpfs/pom.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+  <parent>
+    <groupId>org.apache.bigtop.itest</groupId>
+    <artifactId>smoke-tests</artifactId>
+    <version>0.8.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.bigtop.itest</groupId>
+  <artifactId>httpfs-smoke-execution</artifactId>
+  <version>0.8.0-SNAPSHOT</version>
+  <name>Httpfs smoke test execution</name>
+
+  <properties>
+    <org.apache.maven-dependency-plugin.groupId>org.apache.bigtop.itest</org.apache.maven-dependency-plugin.groupId>
+    <org.apache.maven-dependency-plugin.artifactId>httpfs-smoke</org.apache.maven-dependency-plugin.artifactId>
+    <org.apache.maven-dependency-plugin.version>${httpfs-smoke.version}</org.apache.maven-dependency-plugin.version>
+    <org.apache.maven-dependency-plugin.type>jar</org.apache.maven-dependency-plugin.type>
+
+    <HADOOP_HOME>${env.HADOOP_HOME}</HADOOP_HOME>
+    <HADOOP_CONF_DIR>${env.HADOOP_CONF_DIR}</HADOOP_CONF_DIR>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.bigtop.itest</groupId>
+      <artifactId>httpfs-smoke</artifactId>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <version>1.0</version>
+        <executions>
+          <execution>
+            <id>enforce-property</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <requireProperty>
+                  <property>HADOOP_HOME</property>
+                  <message>HADOOP_HOME env. variable has to be set</message>
+                </requireProperty>
+                <requireProperty>
+                  <property>HADOOP_CONF_DIR</property>
+                  <message>HADOOP_CONF_DIR env. variable has to be set</message>
+                </requireProperty>
+              </rules>
+              <fail>true</fail>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-failsafe-plugin</artifactId>
+        <version>2.11</version>
+        <configuration>
+          <additionalClasspathElements>
+              <additionalClasspathElement>${HADOOP_CONF_DIR}</additionalClasspathElement>
+          </additionalClasspathElements>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-execution/smokes/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-execution/smokes/pom.xml b/bigtop-tests/test-execution/smokes/pom.xml
index 7ccc8bd..1da31b2 100644
--- a/bigtop-tests/test-execution/smokes/pom.xml
+++ b/bigtop-tests/test-execution/smokes/pom.xml
@@ -37,6 +37,7 @@
     <itest-conf.version>${project.version}</itest-conf.version>
     <itest-common.version>${project.version}</itest-common.version>
     <hadoop-smoke.version>${project.version}</hadoop-smoke.version>
+    <httpfs-smoke.version>${project.version}</httpfs-smoke.version>
     <flume-smoke.version>${project.version}</flume-smoke.version>
     <hbase-smoke.version>${project.version}</hbase-smoke.version>
     <hive-smoke.version>${project.version}</hive-smoke.version>
@@ -64,8 +65,8 @@
     <module>giraph</module>
     <module>hue</module>
     <module>crunch</module>
-
     <!--
+    <module>httpfs</module>
     <module>flume</module>
     <module>sqoop</module>
     -->
@@ -154,6 +155,12 @@
         <version>${phoenix-smoke.version}</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.apache.bigtop.itest</groupId>
+        <artifactId>httpfs-smoke</artifactId>
+        <version>${httpfs-smoke.version}</version>
+        <scope>test</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 


Mime
View raw message