Mailing-List: contact deltacloud-dev-help@incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: deltacloud-dev@incubator.apache.org
Received-SPF: pass (nike.apache.org: domain of mfojtik@redhat.com designates
 209.132.183.28 as permitted sender)
Subject: Re: [PATCH] Adds google storage driver for google storage API v.1
Mime-Version: 1.0 (Apple Message framework v1244.3)
Content-Type: text/plain; charset=us-ascii
From: Michal Fojtik <mfojtik@redhat.com>
In-Reply-To: <1316784844-13224-2-git-send-email-marios@redhat.com>
Date: Tue, 4 Oct 2011 13:27:43 +0200
Content-Transfer-Encoding: quoted-printable
Message-Id: <5F0FD545-D336-4DCB-88D4-4C53BE31852F@redhat.com>
References: <1316784844-13224-1-git-send-email-marios@redhat.com>
 <1316784844-13224-2-git-send-email-marios@redhat.com>
To: deltacloud-dev@incubator.apache.org

On Sep 23, 2011, at 3:34 PM, marios@redhat.com wrote:

ACK. I tested it and it works and looks well. Good job marios!.

(just one small minor suggestion inline)

 -- Michal


> From: marios <marios@redhat.com>
>=20
>=20
> Signed-off-by: marios <marios@redhat.com>
> ---
> server/config/drivers/google.yaml                  |    3 +
> .../lib/deltacloud/drivers/google/google_driver.rb |  227 =
++++++++++++++++++++
> server/lib/deltacloud/helpers/blob_stream.rb       |    1 +
> 3 files changed, 231 insertions(+), 0 deletions(-)
> create mode 100644 server/config/drivers/google.yaml
> create mode 100644 =
server/lib/deltacloud/drivers/google/google_driver.rb
>=20
> diff --git a/server/config/drivers/google.yaml =
b/server/config/drivers/google.yaml
> new file mode 100644
> index 0000000..f97d29b
> --- /dev/null
> +++ b/server/config/drivers/google.yaml
> @@ -0,0 +1,3 @@
> +---
> +:google:
> +  :name: Google
> diff --git a/server/lib/deltacloud/drivers/google/google_driver.rb =
b/server/lib/deltacloud/drivers/google/google_driver.rb
> new file mode 100644
> index 0000000..6e1ef01
> --- /dev/null
> +++ b/server/lib/deltacloud/drivers/google/google_driver.rb
> @@ -0,0 +1,227 @@
> +# Licensed to the Apache Software Foundation (ASF) under one or more
> +# contributor license agreements.  See the NOTICE file distributed =
with
> +# this work for additional information regarding copyright ownership. =
 The
> +# ASF licenses this file to you under the Apache License, Version 2.0 =
(the
> +# "License"); you may not use this file except in compliance with the
> +# License.  You may obtain a copy of the License at
> +#
> +#       http://www.apache.org/licenses/LICENSE-2.0
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS, =
WITHOUT
> +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  =
See the
> +# License for the specific language governing permissions and =
limitations
> +# under the License.
> +
> +require 'fog'
> +
> +module Deltacloud
> +  module Drivers
> +    module Google
> +
> +class GoogleDriver < Deltacloud::BaseDriver
> +
> +  def supported_collections; [:buckets]
> +  end
> +
> +  feature :buckets, :bucket_location
> +
> +#--
> +# Buckets
> +#--
> +  def buckets(credentials, opts=3D{})
> +    buckets =3D []
> +    google_client =3D new_client(credentials)
> +    safely do
> +      if opts[:id]
> +        bucket =3D google_client.get_bucket(opts[:id])
> +        buckets << convert_bucket(bucket.body)
> +      else
> +        google_client.get_service.body['Buckets'].each do |bucket|
> +          buckets << Bucket.new({:name =3D> bucket['Name'], :id =3D> =
bucket['Name']})
> +        end

This can look as:

buckets =3D google_client.get_service.body['Buckets'].collect { |bucket| =
Bucket.new(:name =3D> bucket['Name'], :id =3D> bucket['Name']) }

> +      end
> +    end
> +    buckets =3D filter_on(buckets, :id, opts)
> +  end
> +
> +#--
> +# Create bucket - valid values for location {'EU','US'}
> +#--
> +  def create_bucket(credentials, name, opts=3D{})
> +    google_client =3D new_client(credentials)
> +    safely do
> +      bucket_location =3D opts['location']
> +      if (bucket_location && bucket_location.size > 0)
> +        res =3D google_client.put_bucket(name, {"LocationConstraint" =
=3D> opts['location']})
> +      else
> +        google_client.put_bucket(name)
> +      end
> +      #res.status should be eql 200 - but fog will explode if not all =
ok...
> +      Bucket.new({ :id =3D> name,
> +                   :name =3D> name,
> +                   :size =3D> 0,
> +                   :blob_list =3D> [] })
> +    end
> +  end
> +
> +#--
> +# Delete bucket
> +#--
> +  def delete_bucket(credentials, name, opts=3D{})
> +    google_client =3D new_client(credentials)
> +    safely do
> +      google_client.delete_bucket(name)
> +    end
> +  end
> +
> +#--
> +# Blobs
> +#--
> +  def blobs(credentials, opts=3D{})
> +    blobs =3D []
> +    google_client =3D new_client(credentials)
> +    safely do
> +      google_blob =3D google_client.head_object(opts['bucket'], =
opts[:id]).headers
> +      blobs << Blob.new({   :id =3D> opts[:id],
> +                 :bucket =3D> opts['bucket'],
> +                 :content_length =3D> google_blob['Content-Length'],
> +                 :content_type =3D> google_blob['Content-Type'],
> +                 :last_modified =3D> google_blob['Last-Modified'],
> +                 :user_metadata =3D> google_blob.select{|k,v| =
k.match(/^x-goog-meta-/i)}
> +              })
> +
> +    end
> +    blobs
> +  end
> +
> +  def blob_data(credentials, bucket_id, blob_id, opts=3D{})
> +    google_client =3D new_client(credentials)
> +    safely do
> +      google_client.get_object(bucket_id, blob_id) do |chunk|
> +        yield chunk
> +      end
> +    end
> +  end
> +
> +#--
> +# Create Blob
> +#--
> +  def create_blob(credentials, bucket_id, blob_id, blob_data, =
opts=3D{})
> +    google_client =3D new_client(credentials)
> +    safely do
> +      dcloud_blob_metadata =3D =
BlobHelper::extract_blob_metadata_hash(opts)
> +      BlobHelper::rename_metadata_headers(opts, 'x-goog-meta-')
> +      opts['Content-Type'] =3D blob_data[:type]
> +      google_client.put_object(bucket_id, blob_id, =
blob_data[:tempfile], opts)
> +      Blob.new({ :id =3D> blob_id,
> +                 :bucket =3D> bucket_id,
> +                 :content_length =3D> =
File.size(blob_data[:tempfile]).to_s,
> +                 :content_type =3D> blob_data[:type],
> +                 :last_modified =3D> "",
> +                 :user_metadata =3D> dcloud_blob_metadata  })
> +    end
> +  end
> +
> +  #params: =
{:user,:password,:bucket,:blob,:content_type,:content_length,:metadata}
> +  def blob_stream_connection(params)
> +#fog/storage/requests/google/put_object.rb:put_object calls  =
'request' =3D=3D=3D>
> +#fog/storage/google:request calls 'signature' =3D=3D=3D> =
#fog/storage/google:signature and then
> +#the actual request method is in fog/core/connection:9
> +#params[:headers]['Date'] =3D Fog::Time.now.to_date_header
> +    client =3D Fog::Storage.new({:provider =3D> :google, =
:google_storage_access_key_id =3D> params[:user],
> +                               :google_storage_secret_access_key =3D> =
params[:password]})
> +    google_request_uri =3D =
"https://#{client.instance_variable_get(:@host)}"
> +    uri =3D URI.parse(google_request_uri)
> +    conn_params =3D {} # build hash for the Fog signature method
> +    conn_params[:headers] =3D {} #put the metadata here
> +    conn_params[:host] =3D uri.host
> +    conn_params[:path] =3D =
"#{params[:bucket]}/#{CGI.escape(params[:blob])}"
> +    conn_params[:method] =3D "PUT"
> +    timestamp =3D Fog::Time.now.to_date_header
> +    conn_params[:headers]['Date'] =3D timestamp
> +    conn_params[:headers]['Content-Type'] =3D params[:content_type]
> +    metadata =3D params[:metadata] || {}
> +    BlobHelper::rename_metadata_headers(metadata, 'x-goog-meta-')
> +    metadata.each{|k,v| conn_params[:headers][k]=3Dv}
> +    auth_string =3D "GOOG1 =
#{params[:user]}:#{client.signature(conn_params)}"
> +
> +    http =3D Net::HTTP.new(uri.host, uri.port)
> +    http.use_ssl =3D true
> +    http.verify_mode =3D OpenSSL::SSL::VERIFY_NONE
> +    request =3D Net::HTTP::Put.new("/#{conn_params[:path]}")
> +    request['Host'] =3D conn_params[:host]
> +    request['Date'] =3D conn_params[:headers]['Date']
> +    request['Content-Type'] =3D conn_params[:headers]['Content-Type']
> +    request['Content-Length'] =3D params[:content_length]
> +    request['Authorization'] =3D auth_string
> +    metadata.each{|k,v| request[k] =3D v}
> +    return http, request
> +  end
> +
> +#--
> +# Delete Blob
> +#--
> +  def delete_blob(credentials, bucket_id, blob_id, opts=3D{})
> +    google_client =3D new_client(credentials)
> +    safely do
> +      google_client.delete_object(bucket_id, blob_id)
> +    end
> +  end
> +
> +#-
> +# Blob Metadada
> +#-
> +  def blob_metadata(credentials, opts =3D {})
> +    google_client =3D new_client(credentials)
> +    safely do
> +      google_blob =3D google_client.head_object(opts['bucket'], =
opts[:id]).headers
> +      meta_hash =3D google_blob.inject({}){|result, (k,v)| =
result[k]=3Dv if k=3D~/^x-goog-meta-/i ; result}
> +      meta_hash.gsub_keys("x-goog-meta-", "")
> +    end
> +  end
> +
> +#-
> +# Update Blob Metadata
> +#-
> +  def update_blob_metadata(credentials, opts=3D{})
> +    google_client =3D new_client(credentials)
> +    safely do
> +      meta_hash =3D =
BlobHelper::rename_metadata_headers(opts['meta_hash'], 'x-goog-meta-')
> +      options =3D {'x-goog-metadata-directive'=3D>'REPLACE'}
> +      options.merge!(meta_hash)
> +      bucket =3D opts['bucket']
> +      blob =3D opts[:id]
> +      google_client.copy_object(bucket, blob, bucket, blob, options) =
#source,source,target,target,options
> +      meta_hash.gsub_keys("x-goog-meta-", "")
> +    end
> +  end
> +
> +
> +  private
> +
> +  def new_client(credentials)
> +    safely do
> +      Fog::Storage.new({ :provider =3D> :google, =
:google_storage_access_key_id =3D> credentials.user,
> +                         :google_storage_secret_access_key =3D> =
credentials.password})
> +
> +    end
> +  end
> +
> +  def convert_bucket(bucket)
> +    blob_list =3D []
> +    bucket['Contents'].each do |blob|
> +      blob_list << blob['Key']
> +    end
> +    Bucket.new({    :id =3D> bucket['Name'],
> +                    :name =3D> bucket['Name'],
> +                    :size =3D> blob_list.size,
> +                    :blob_list =3D> blob_list
> +                  })
> +  end
> +
> +end
> +
> +    end
> +  end
> +end
> diff --git a/server/lib/deltacloud/helpers/blob_stream.rb =
b/server/lib/deltacloud/helpers/blob_stream.rb
> index 93f671c..07b9566 100644
> --- a/server/lib/deltacloud/helpers/blob_stream.rb
> +++ b/server/lib/deltacloud/helpers/blob_stream.rb
> @@ -104,6 +104,7 @@ module Thin
> end
>=20
> require 'net/http'
> +require 'net/https'
> #monkey patch for Net:HTTP
> module Net
>   class HTTP
> --=20
> 1.7.3.4
>=20

------------------------------------------------------
Michal Fojtik, mfojtik@redhat.com
Deltacloud API: http://deltacloud.org