incubator-cloudstack-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From David Nalley <da...@gnsa.us>
Subject HDFS and CloudStack? was: [SCM] CloudStack OSS branch s3-hdfs
Date Tue, 22 May 2012 19:58:28 GMT
Hi Chiradeep -

I just saw this branch created - which has a fascinating name - sadly
I can't find any docs that talk about Hadoop or HDFS on the wiki
(there is a parenthetical statement on the wiki that says 'like
Hadoop' but nothing much past that) Can you tell us about what your
plan is here, objectives? Functional spec exist for any of this work?

--David



-------- Original Message --------
Subject: [SCM] CloudStack OSS branch s3-hdfs created.
15dd6a25254d83076cd362d65a728fee8681a4b4
Date: Tue, 22 May 2012 15:11:14 -0400
From: git@cloud.com <git@cloud.com>
To: SourceForge <cloudstack-commits@lists.sourceforge.net>

This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "CloudStack OSS".

The branch, s3-hdfs has been created
       at  15dd6a25254d83076cd362d65a728fee8681a4b4 (commit)

- Log -----------------------------------------------------------------

http://git.cloud.com/cgit/cloudstack-oss/commit/?id=15dd6a25254d83076cd362d65a728fee8681a4b4

commit 15dd6a25254d83076cd362d65a728fee8681a4b4
Author: Chiradeep Vittal <chiradeep@cloud.com>
Date:   Tue May 22 12:44:56 2012 -0700

   Handle HDFS as the backend

diff --git a/awsapi/.classpath b/awsapi/.classpath
index ab9174c..18f729d 100644
--- a/awsapi/.classpath
+++ b/awsapi/.classpath
@@ -4,64 +4,5 @@
       <classpathentry kind="src" path="test"/>
       <classpathentry kind="con"
path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
       <classpathentry combineaccessrules="false" kind="src" path="/deps"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/antlr-2.7.6.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/apache-log4j-extras-1.0.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axiom-api-1.2.8.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axiom-impl-1.2.8.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axis2-1.5.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axis2-adb-1.5.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axis2-ant-plugin-1.5.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axis2-jaxbri-1.5.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axis2-jaxws-1.5.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axis2-jibx-1.5.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axis2-json-1.5.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axis2-kernel-1.5.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axis2-transport-http-1.5.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/axis2-transport-local-1.5.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/cloud-gson.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/commons-codec-1.4.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/commons-collections-3.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/commons-fileupload-1.2.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/commons-httpclient-3.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/commons-io-1.4.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/commons-logging-1.1.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/dom4j-1.6.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/hibernate3.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/httpcore-4.0.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/javassist-3.9.0.GA.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/jaxb-api-2.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/jaxb-impl-2.1.7.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/jaxb-xjc-2.1.7.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/jsch-0.1.42.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/json_simple-1.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/jta-1.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/junit-4.8.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/log4j-1.2.15.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/mail-1.4.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/mysql-connector-java-5.1.7-bin.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/neethi-2.0.4.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/slf4j-api-1.5.11.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/slf4j-jdk14-1.5.11.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/woden-api-1.0M8.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/woden-impl-dom-1.0M8.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/xercesImpl.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/xml-apis.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/XmlSchema-1.4.3.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/bcprov-jdk16-145.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/commons-collections-3.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/joda-time-1.5.2.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/opensaml-1.1.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/opensaml-2.2.3.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/openws-1.2.2.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/rampart-core-1.5.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/rampart-policy-1.5.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/rampart-trust-1.5.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/slf4j-api-1.5.11.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/slf4j-jdk14-1.5.11.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/velocity-1.5.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/wss4j-1.5.8.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/xmlsec-1.4.2.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/awsapi-lib/rampart-lib/xmltooling-1.2.0.jar"/>
-       <classpathentry kind="lib"
path="/Users/john1/qgit/deps/cloudbridge-lib/servlet-api.jar"/>
       <classpathentry kind="output" path="bin"/>
 </classpath>
diff --git a/awsapi/src/com/cloud/bridge/model/SHost.java
b/awsapi/src/com/cloud/bridge/model/SHost.java
index 2d26968..864956c 100644
--- a/awsapi/src/com/cloud/bridge/model/SHost.java
+++ b/awsapi/src/com/cloud/bridge/model/SHost.java
@@ -27,6 +27,8 @@ public class SHost implements Serializable {


       public static final int STORAGE_HOST_TYPE_LOCAL = 0;

       public static final int STORAGE_HOST_TYPE_NFS = 1;

+       public static final int STORAGE_HOST_TYPE_HDFS = 2;

+



       private Long id;



diff --git
a/awsapi/src/com/cloud/bridge/service/controller/s3/ServiceProvider.java
b/awsapi/src/com/cloud/bridge/service/controller/s3/ServiceProvider.java
index 18acb1a..6b3e24e 100644
--- a/awsapi/src/com/cloud/bridge/service/controller/s3/ServiceProvider.java
+++ b/awsapi/src/com/cloud/bridge/service/controller/s3/ServiceProvider.java
@@ -26,6 +26,10 @@ import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
 import java.sql.SQLException;
 import java.util.HashMap;
 import java.util.Map;
@@ -34,6 +38,7 @@ import java.util.Timer;
 import java.util.TimerTask;

 import org.apache.axis2.AxisFault;
+import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
 import org.apache.log4j.Logger;
 import org.apache.log4j.xml.DOMConfigurator;
 import org.hibernate.SessionException;
@@ -233,7 +238,12 @@ public class ServiceProvider {
               PersistContext.flush();

               String localStorageRoot =
properties.getProperty("storage.root");
-               if (localStorageRoot != null)
setupLocalStorage(localStorageRoot);
+               if (localStorageRoot != null) {
+                   if (localStorageRoot.startsWith("hdfs"))
+                           setupHdfsStorage(localStorageRoot);
+                   else
+                       setupLocalStorage(localStorageRoot);
+               }

               multipartDir =
properties.getProperty("storage.multipartDir");

@@ -243,7 +253,29 @@ public class ServiceProvider {
                       logger.info("ServiceProvider initialized");
       }

-       private void loadStartupProperties() {
+       private void setupHdfsStorage(String storageRoot) {
+           URI uri;
+        try {
+            uri = new URI(storageRoot);
+        } catch (URISyntaxException e) {
+           throw new ConfigurationException("Invalid hdfs URI: " +
storageRoot + " should be hdfs://host/path");
+        }
+           SHostDao shostDao = new SHostDao();
+        SHost shost = shostDao.getLocalStorageHost(mhost.getId(),
storageRoot);
+        if(shost == null) {
+            shost = new SHost();
+            shost.setMhost(mhost);
+            mhost.getLocalSHosts().add(shost);
+            shost.setHostType(SHost.STORAGE_HOST_TYPE_HDFS);
+            shost.setHost(uri.getHost());
+            //shost.setExportRoot(url.getPath());
+            shost.setExportRoot(storageRoot);
+            PersistContext.getSession().save(shost);
+        }
+
+    }
+
+    private void loadStartupProperties() {
               File propertiesFile =
ConfigurationHelper.findConfigurationFile("cloud-bridge.properties");
               properties = new Properties();
               if(propertiesFile != null) {
diff --git a/awsapi/src/com/cloud/bridge/service/core/s3/S3Engine.java
b/awsapi/src/com/cloud/bridge/service/core/s3/S3Engine.java
index 4b04d3d..4af18a3 100644
--- a/awsapi/src/com/cloud/bridge/service/core/s3/S3Engine.java
+++ b/awsapi/src/com/cloud/bridge/service/core/s3/S3Engine.java
@@ -18,6 +18,8 @@ package com.cloud.bridge.service.core.s3;
 import java.io.IOException;

 import java.io.InputStream;

 import java.io.OutputStream;

+import java.net.URI;

+import java.net.URISyntaxException;

 import java.sql.SQLException;

 import java.util.ArrayList;

 import java.util.Calendar;

@@ -39,6 +41,7 @@ import org.hibernate.LockMode;
 import org.hibernate.Session;

 import org.json.simple.parser.ParseException;



+import com.cloud.bridge.io.HdfsBucketAdapter;

 import com.cloud.bridge.io.S3FileSystemBucketAdapter;

 import com.cloud.bridge.model.MHost;

 import com.cloud.bridge.model.MHostMount;

@@ -94,6 +97,14 @@ public class S3Engine {


    public S3Engine() {

       bucketAdapters.put(SHost.STORAGE_HOST_TYPE_LOCAL, new
S3FileSystemBucketAdapter());

+       HdfsBucketAdapter hdfsAdapter = new HdfsBucketAdapter();

+       try {

+           hdfsAdapter.initialize();

+            bucketAdapters.put(SHost.STORAGE_HOST_TYPE_HDFS, hdfsAdapter);

+       }catch (IOException e ){

+           return;

+       }

+

    }





@@ -1375,7 +1386,16 @@ public class S3Engine {
               if(shost.getHostType() == SHost.STORAGE_HOST_TYPE_LOCAL) {

                       return new OrderedPair<SHost, String>(shost,
shost.getExportRoot());

               }

-

+

+               if(shost.getHostType() == SHost.STORAGE_HOST_TYPE_HDFS ) {

+                   try {

+                       URI uri = new URI(shost.getExportRoot());

+                       return new OrderedPair<SHost, String>(shost,
uri.getPath());

+                   } catch (URISyntaxException use) {

+                       throw new InternalErrorException("Could not
determine storage path for HDFS");

+                   }

+               }

+

               MHostMount mount =
mountDao.getHostMount(ServiceProvider.getInstance().getManagementHostId(),
shost.getId());

               if(mount != null) {

                       return new OrderedPair<SHost, String>(shost,
mount.getMountPath());

diff --git a/deps/.classpath b/deps/.classpath
index f704a05..66687cb 100755
--- a/deps/.classpath
+++ b/deps/.classpath
@@ -1,65 +1,71 @@
-<?xml version="1.0" encoding="UTF-8"?>

-<classpath>

-       <classpathentry kind="src" path="XenServerJava"/>

-       <classpathentry exported="true" kind="con"
path="org.eclipse.jdt.launching.JRE_CONTAINER"/>

-       <classpathentry exported="true" kind="lib" path="cloud-axis.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-backport-util-concurrent-3.0.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-bcprov-jdk16-1.45.jar"/>

-       <classpathentry exported="true" kind="lib" path="cloud-cglib.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-commons-collections-3.2.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-commons-discovery.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-commons-logging-1.1.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-ehcache.jar"/>

-       <classpathentry exported="true" kind="lib" path="cloud-email.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-httpcore-4.0.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-jasypt-1.8.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-jsch-0.1.42.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-jstl-1.2.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-libvirt-0.4.5.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-log4j.jar"
sourcepath="/home/dev/thirdparty/apache-log4j-1.2.16/src/main/java"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-mysql-connector-java-5.1.7-bin.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-servlet-api.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-trilead-ssh2-build213.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-ws-commons-util-1.0.2.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-xenserver-5.6.100-1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-xmlrpc-client-3.1.3.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-xmlrpc-common-3.1.3.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-xstream-1.3.1.jar"/>

-       <classpathentry exported="true" kind="lib" path="jetty-6.1.26.jar"/>

-       <classpathentry exported="true" kind="lib"
path="jetty-util-6.1.26.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-log4j-extras.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-iControl.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-manageontap.jar"/>

-       <classpathentry exported="true" kind="lib"
path="vmware-apputils.jar"/>

-       <classpathentry exported="true" kind="lib"
path="vmware-lib-jaxrpc.jar"/>

-       <classpathentry exported="true" kind="lib" path="vmware-vim.jar"/>

-       <classpathentry exported="true" kind="lib" path="vmware-vim25.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-jnetpcap.jar"/>

-       <classpathentry exported="true" kind="lib" path="cloud-junit.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-commons-codec-1.5.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-commons-dbcp-1.4.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-commons-pool-1.5.6.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-google-gson-1.7.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-commons-httpclient-3.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-netscaler.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-netscaler-sdx.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-commons-configuration-1.8.jar"/>

-       <classpathentry exported="true" kind="lib"
path="cloud-commons-lang-2.6.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axiom-api-1.2.8.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axiom-impl-1.2.8.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-1.5.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-adb-1.5.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-ant-plugin-1.5.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-jaxbri-1.5.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-jaxws-1.5.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-jibx-1.5.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-json-1.5.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-kernel-1.5.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-transport-http-1.5.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-transport-local-1.5.1.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/hibernate3.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/neethi-2.0.4.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/rampart-lib/wss4j-1.5.8.jar"/>

-       <classpathentry exported="true" kind="lib"
path="awsapi-lib/json_simple-1.1.jar"/>

-       <classpathentry kind="output" path="bin"/>

-</classpath>

+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+       <classpathentry kind="src" path="XenServerJava"/>
+       <classpathentry exported="true" kind="con"
path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+       <classpathentry exported="true" kind="lib" path="cloud-axis.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-backport-util-concurrent-3.0.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-bcprov-jdk16-1.45.jar"/>
+       <classpathentry exported="true" kind="lib" path="cloud-cglib.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-commons-collections-3.2.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-commons-discovery.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-commons-logging-1.1.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-ehcache.jar"/>
+       <classpathentry exported="true" kind="lib" path="cloud-email.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-httpcore-4.0.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-jasypt-1.8.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-jsch-0.1.42.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-jstl-1.2.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-libvirt-0.4.5.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-log4j.jar"
sourcepath="/home/dev/thirdparty/apache-log4j-1.2.16/src/main/java"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-mysql-connector-java-5.1.7-bin.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-servlet-api.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-trilead-ssh2-build213.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-ws-commons-util-1.0.2.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-xenserver-5.6.100-1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-xmlrpc-client-3.1.3.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-xmlrpc-common-3.1.3.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-xstream-1.3.1.jar"/>
+       <classpathentry exported="true" kind="lib" path="jetty-6.1.26.jar"/>
+       <classpathentry exported="true" kind="lib"
path="jetty-util-6.1.26.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-log4j-extras.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-iControl.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-manageontap.jar"/>
+       <classpathentry exported="true" kind="lib"
path="vmware-apputils.jar"/>
+       <classpathentry exported="true" kind="lib"
path="vmware-lib-jaxrpc.jar"/>
+       <classpathentry exported="true" kind="lib" path="vmware-vim.jar"/>
+       <classpathentry exported="true" kind="lib" path="vmware-vim25.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-jnetpcap.jar"/>
+       <classpathentry exported="true" kind="lib" path="cloud-junit.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-commons-codec-1.5.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-commons-dbcp-1.4.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-commons-pool-1.5.6.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-google-gson-1.7.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-commons-httpclient-3.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-netscaler.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-netscaler-sdx.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-commons-configuration-1.8.jar"/>
+       <classpathentry exported="true" kind="lib"
path="cloud-commons-lang-2.6.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axiom-api-1.2.8.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axiom-impl-1.2.8.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-1.5.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-adb-1.5.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-ant-plugin-1.5.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-jaxbri-1.5.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-jaxws-1.5.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-jibx-1.5.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-json-1.5.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-kernel-1.5.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-transport-http-1.5.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/axis2-transport-local-1.5.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/hibernate3.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/neethi-2.0.4.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/rampart-lib/wss4j-1.5.8.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/json_simple-1.1.jar"/>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/hadoop-hdfs-0.23.1.jar"
sourcepath="/Users/chiradeep/source/hadoop-hdfs-0.23.1-sources.jar">
+               <attributes>
+                       <attribute name="javadoc_location"
value="jar:file:/Users/chiradeep/source/hadoop-hdfs-0.23.1-sources.jar!/"/>
+               </attributes>
+       </classpathentry>
+       <classpathentry exported="true" kind="lib"
path="awsapi-lib/hadoop-common-0.23.1.jar"
sourcepath="/Users/chiradeep/source/hadoop-common-0.23.1-sources.jar"/>
+       <classpathentry kind="output" path="bin"/>
+</classpath>
diff --git a/deps/awsapi-lib/xercesImpl.jar b/deps/awsapi-lib/xercesImpl.jar
index 3b351f6..0aaa990 100644
Binary files a/deps/awsapi-lib/xercesImpl.jar and
b/deps/awsapi-lib/xercesImpl.jar differ
diff --git a/deps/awsapi-lib/xml-apis.jar b/deps/awsapi-lib/xml-apis.jar
index b338fb6..4673346 100644
Binary files a/deps/awsapi-lib/xml-apis.jar and
b/deps/awsapi-lib/xml-apis.jar differ


http://git.cloud.com/cgit/cloudstack-oss/commit/?id=52c59e2df65b6b7345a8228dbe3a9945644cf4a8

commit 52c59e2df65b6b7345a8228dbe3a9945644cf4a8
Author: Chiradeep Vittal <chiradeep@cloud.com>
Date:   Tue May 22 12:44:02 2012 -0700

   Initial commit of HDFS-related files

diff --git a/awsapi/src/com/cloud/bridge/io/HdfsBucketAdapter.java
b/awsapi/src/com/cloud/bridge/io/HdfsBucketAdapter.java
new file mode 100644
index 0000000..7a10f15
--- /dev/null
+++ b/awsapi/src/com/cloud/bridge/io/HdfsBucketAdapter.java
@@ -0,0 +1,117 @@
+package com.cloud.bridge.io;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import javax.activation.DataHandler;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.bridge.service.core.s3.S3BucketAdapter;
+import com.cloud.bridge.service.core.s3.S3MultipartPart;
+import com.cloud.bridge.service.exception.InternalErrorException;
+import com.cloud.bridge.service.exception.OutOfStorageException;
+import com.cloud.bridge.service.exception.UnsupportedException;
+import com.cloud.bridge.util.OrderedPair;
+
+public class HdfsBucketAdapter implements S3BucketAdapter {
+    protected final static Logger logger =
Logger.getLogger(HdfsBucketAdapter.class);
+
+    HdfsClient _client = new HdfsClient();
+
+    public HdfsBucketAdapter() {
+    }
+
+    public void initialize() throws IOException {
+        _client.initialize();
+    }
+
+    @Override
+    public void createContainer(String mountedRoot, String bucket) {
+        String dir = getBucketFolderDir(mountedRoot, bucket);
+        try {
+            _client.mkdir(dir);
+        } catch (IOException e) {
+            throw new OutOfStorageException("Unable to create container
" + bucket, e);
+        }
+
+    }
+
+    @Override
+    public void deleteContainer(String mountedRoot, String bucket) {
+        String dir = getBucketFolderDir(mountedRoot, bucket);
+        try {
+            _client.deleteFile(dir);
+        } catch (IOException e) {
+            throw new OutOfStorageException("Unable to delete " + dir +
" for bucket " + bucket);
+        }
+
+    }
+
+    @Override
+    public String getBucketFolderDir(String mountedRoot, String bucket) {
+        String bucketFolder = getBucketFolderName(bucket);
+        String dir;
+        String separator = "" + File.separatorChar;
+        if(!mountedRoot.endsWith(separator))
+            dir = mountedRoot + separator + bucketFolder;
+        else
+            dir = mountedRoot + bucketFolder;
+
+        return dir;
+    }
+
+    @Override
+    public String saveObject(InputStream is, String mountedRoot, String
bucket, String fileName) {
+        String name = getBucketFolderDir(mountedRoot, bucket) +
File.separatorChar + fileName;
+        try {
+            return _client.saveFile(is, name);
+        } catch (IOException e) {
+            throw new OutOfStorageException(e);
+        }
+    }
+
+    @Override
+    public DataHandler loadObject(String mountedRoot, String bucket,
String fileName) {
+        String src = getBucketFolderDir(mountedRoot, bucket) +
File.separatorChar + fileName;
+        try {
+            return _client.readFile(src);
+        } catch (IOException e) {
+           throw new OutOfStorageException(e);
+        }
+    }
+
+    @Override
+    public DataHandler loadObjectRange(String mountedRoot, String
bucket, String fileName, long startPos, long endPos) {
+        String src = getBucketFolderDir(mountedRoot, bucket) +
File.separatorChar + fileName;
+        return _client.readRange(src, startPos, endPos);
+    }
+
+    @Override
+    public void deleteObject(String mountedRoot, String bucket, String
fileName) {
+        String src = getBucketFolderDir(mountedRoot, bucket) +
File.separatorChar + fileName;
+        try {
+            _client.deleteFile(src);
+        } catch (IOException e) {
+            throw new InternalErrorException("Unable to delete object "
+ src, e);
+        }
+
+    }
+
+    @Override
+    public OrderedPair<String, Long> concatentateObjects(String
mountedRoot, String destBucket, String fileName, String sourceBucket,
S3MultipartPart[] parts, OutputStream client) {
+        throw new UnsupportedException("HDFS backing store does not
support concatenate yet");
+        //TODO
+    }
+
+    private String getBucketFolderName(String bucket) {
+        // temporary
+        String name = bucket.replace(' ', '_');
+        name = bucket.replace('\\', '-');
+        name = bucket.replace('/', '-');
+
+        return name;
+    }
+}
diff --git a/awsapi/src/com/cloud/bridge/io/HdfsClient.java
b/awsapi/src/com/cloud/bridge/io/HdfsClient.java
new file mode 100644
index 0000000..0079a53
--- /dev/null
+++ b/awsapi/src/com/cloud/bridge/io/HdfsClient.java
@@ -0,0 +1,175 @@
+package com.cloud.bridge.io;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.MalformedURLException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import javax.activation.DataHandler;
+import javax.activation.DataSource;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+import com.cloud.bridge.service.exception.FileNotExistException;
+import com.cloud.bridge.service.exception.InternalErrorException;
+import com.cloud.bridge.service.exception.OutOfStorageException;
+import com.cloud.bridge.util.ConfigurationHelper;
+import com.cloud.bridge.util.StringHelper;
+
+public class HdfsClient {
+    private static final class HdfsDataSource implements DataSource {
+        InputStream istream;
+
+        public HdfsDataSource(InputStream in) {
+           istream = in;
+        }
+
+        @Override
+        public OutputStream getOutputStream() throws IOException {
+            return null; //TODO
+        }
+
+        @Override
+        public String getName() {
+            return this.getClass().getName();
+        }
+
+        @Override
+        public InputStream getInputStream() throws IOException {
+            return istream;
+        }
+
+        @Override
+        public String getContentType() {
+            return "application/octet-stream";
+        }
+    }
+    protected final static Logger logger =
Logger.getLogger(HdfsClient.class);
+
+    Configuration conf = new Configuration();
+    FileSystem fileSystem;
+    boolean initialized = false;
+
+
+    public HdfsClient() {
+
+    }
+
+    public void initialize() throws IOException {
+        if (initialized) {
+            return;
+        }
+        File coreCfg =
ConfigurationHelper.findConfigurationFile("core-site.xml");
+        File siteCfg =
ConfigurationHelper.findConfigurationFile("hdfs-site.xml");
+        conf.addResource(coreCfg.getAbsolutePath());
+        conf.addResource(siteCfg.getAbsolutePath());
+        fileSystem = FileSystem.get(conf);
+        initialized = true;
+
+    }
+
+    public String saveFile(InputStream is, String dest) throws
IOException {
+
+        Path path = new Path(dest);
+
+        FSDataOutputStream fos = null;
+        MessageDigest md5 = null;
+
+        try {
+            md5 = MessageDigest.getInstance("MD5");
+        } catch (NoSuchAlgorithmException e) {
+            throw new InternalErrorException("Unable to get MD5
MessageDigest", e);
+        }
+
+        try {
+            // -> when versioning is off we need to rewrite the file
contents
+            fileSystem.delete(path, true);
+            fos = fileSystem.create(path);
+
+            byte[] buffer = new byte[4096];
+            int len = 0;
+            while( (len = is.read(buffer)) > 0) {
+                fos.write(buffer, 0, len);
+                md5.update(buffer, 0, len);
+
+            }
+            //Convert MD5 digest to (lowercase) hex String
+            return StringHelper.toHexString(md5.digest());
+
+        }
+        catch(IOException e) {
+            throw new OutOfStorageException(e);
+        }
+        finally {
+            try {
+                if (null != fos) fos.close();
+            }
+            catch( Exception e ) {
+                logger.error("Can't close FileOutputStream " +
e.getMessage(), e);
+            }
+        }
+
+    }
+
+    public DataHandler readFile(String file) throws IOException {
+
+        Path path = new Path(file);
+        if (!fileSystem.exists(path)) {
+            throw new FileNotExistException("Unable to open underlying
object file");
+        }
+
+        FSDataInputStream in = fileSystem.open(path);
+
+        return new DataHandler(new HdfsDataSource(in));
+    }
+
+    public DataHandler readRange(String file, long startPos, long endPos) {
+        try {
+            DataSource ds = new HdfsRangeDataSource(fileSystem, file,
startPos, endPos);
+            return new DataHandler(ds);
+        } catch(IOException e) {
+            throw new FileNotExistException("Unable to open underlying
object file");
+        }
+    }
+
+    public void deleteFile(String file) throws IOException {
+
+        Path path = new Path(file);
+        if (!fileSystem.exists(path)) {
+            throw new FileNotExistException("File " + file + " does not
exist");
+        }
+        fileSystem.delete(path, true);
+
+    }
+
+    public void mkdir(String dir) throws IOException {
+
+        Path path = new Path(dir);
+        if (fileSystem.exists(path)) {
+            return;
+        }
+
+        fileSystem.mkdirs(path);
+
+    }
+
+    public void createFile(String fileName) throws IOException {
+
+        Path path = new Path(fileName);
+        if (fileSystem.exists(path)) {
+            return;
+        }
+
+        fileSystem.create(path);
+
+    }
+
+
+}
diff --git a/awsapi/src/com/cloud/bridge/io/HdfsRangeDataSource.java
b/awsapi/src/com/cloud/bridge/io/HdfsRangeDataSource.java
new file mode 100644
index 0000000..0f585ff
--- /dev/null
+++ b/awsapi/src/com/cloud/bridge/io/HdfsRangeDataSource.java
@@ -0,0 +1,60 @@
+/*

+ * Copyright (C) 2011 Citrix Systems, Inc.  All rights reserved.

+ *

+ * Licensed under the Apache License, Version 2.0 (the "License");

+ * you may not use this file except in compliance with the License.

+ * You may obtain a copy of the License at

+ *

+ *      http://www.apache.org/licenses/LICENSE-2.0

+ *

+ * Unless required by applicable law or agreed to in writing, software

+ * distributed under the License is distributed on an "AS IS" BASIS,

+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+ * See the License for the specific language governing permissions and

+ * limitations under the License.

+ */

+package com.cloud.bridge.io;

+

+import java.io.IOException;

+import java.io.InputStream;

+import java.io.OutputStream;

+

+import javax.activation.DataSource;

+

+import org.apache.hadoop.fs.FileSystem;

+import org.apache.hadoop.fs.Path;

+import org.apache.log4j.Logger;

+

+

+public class HdfsRangeDataSource implements DataSource {

+    protected final static Logger logger =
Logger.getLogger(HdfsRangeDataSource.class);

+

+       private HdfsRangeInputStream is;

+

+       public HdfsRangeDataSource(FileSystem fs, String src, long
startPos, long endPos) throws IOException {

+               is = new HdfsRangeInputStream(fs, new Path(src),
startPos, endPos);

+       }

+

+       @Override

+       public String getContentType() {

+               assert(false);

+               return null;

+       }

+

+       @Override

+       public InputStream getInputStream() throws IOException {

+               return is;

+       }

+

+       @Override

+       public String getName() {

+               assert(false);

+               return null;

+       }

+

+       @Override

+       public OutputStream getOutputStream() throws IOException {

+               assert(false);

+               return null;

+       }

+}

diff --git a/awsapi/src/com/cloud/bridge/io/HdfsRangeInputStream.java
b/awsapi/src/com/cloud/bridge/io/HdfsRangeInputStream.java
new file mode 100644
index 0000000..1b4d2e2
--- /dev/null
+++ b/awsapi/src/com/cloud/bridge/io/HdfsRangeInputStream.java
@@ -0,0 +1,102 @@
+/*

+ * Copyright (C) 2011 Citrix Systems, Inc.  All rights reserved.

+ *

+ * Licensed under the Apache License, Version 2.0 (the "License");

+ * you may not use this file except in compliance with the License.

+ * You may obtain a copy of the License at

+ *

+ *      http://www.apache.org/licenses/LICENSE-2.0

+ *

+ * Unless required by applicable law or agreed to in writing, software

+ * distributed under the License is distributed on an "AS IS" BASIS,

+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+ * See the License for the specific language governing permissions and

+ * limitations under the License.

+ */

+package com.cloud.bridge.io;

+

+import java.io.File;

+import java.io.IOException;

+import java.io.InputStream;

+

+import org.apache.hadoop.fs.FSDataInputStream;

+import org.apache.hadoop.fs.FileStatus;

+import org.apache.hadoop.fs.FileSystem;

+import org.apache.hadoop.fs.Path;

+

+/**

+ * @author Kelven Yang

+ */

+public class HdfsRangeInputStream extends InputStream {

+       private FSDataInputStream in;

+       private long curPos;

+       private long endPos;

+       private long fileLength;

+

+       public HdfsRangeInputStream(FileSystem hdfs, Path src, long
startPos, long endPos) throws IOException {

+               FileStatus status = hdfs.getFileStatus(src);

+               fileLength = status.getLen();

+

+               if(startPos > fileLength)

+                       startPos = fileLength;

+

+               if(endPos > fileLength)

+                       endPos = fileLength;

+

+               if(startPos > endPos)

+                       throw new IllegalArgumentException("Invalid file
range " + startPos + "-" + endPos);

+

+               this.curPos = startPos;

+               this.endPos = endPos;

+               in = hdfs.open(src);

+

+               in.seek(startPos);

+       }

+

+       @Override

+       public int available() throws IOException {

+               return (int)(endPos - curPos);

+       }

+

+       @Override

+       public int read() throws IOException {

+               if(available() > 0) {

+                       int value = in.read();

+                       curPos++;

+                       return value;

+               }

+               return -1;

+       }

+

+       @Override

+       public int read(byte[] b) throws IOException {

+               return read(b, 0, b.length);

+       }

+

+       @Override

+       public int read(byte[] b, int off, int len) throws IOException {

+               int bytesToRead = Math.min(len, available());

+               if(bytesToRead == 0)

+                       return -1;

+

+               int bytesRead = in.read(b, off, bytesToRead);

+               if(bytesRead < 0)

+                       return -1;

+

+               curPos += bytesRead;

+               return bytesRead;

+       }

+

+       @Override

+       public long skip(long n) throws IOException {

+               long skipped = Math.min(n, available());

+               in.skipBytes((int)skipped);

+               curPos += skipped;

+               return skipped;

+       }

+

+       @Override

+       public void close() throws IOException {

+               in.close();

+       }

+}

diff --git a/deps/awsapi-lib/hadoop-common-0.23.1.jar
b/deps/awsapi-lib/hadoop-common-0.23.1.jar
new file mode 100755
index 0000000..c73f20c
Binary files /dev/null and b/deps/awsapi-lib/hadoop-common-0.23.1.jar differ
diff --git a/deps/awsapi-lib/hadoop-hdfs-0.23.1.jar
b/deps/awsapi-lib/hadoop-hdfs-0.23.1.jar
new file mode 100755
index 0000000..e3bb8c3
Binary files /dev/null and b/deps/awsapi-lib/hadoop-hdfs-0.23.1.jar differ

-----------------------------------------------------------------------


hooks/post-receive
--
CloudStack OSS

Mime
View raw message