Mailing-List: contact user-help@hbase.apache.org; run by ezmlm
Precedence: bulk
Reply-To: user@hbase.apache.org
Received-SPF: pass (athena.apache.org: domain of ncanis@gmail.com designates
 209.85.216.41 as permitted sender)
DomainKey-Signature: a=rsa-sha1; c=nofws;
        d=gmail.com; s=gamma;
        h=mime-version:in-reply-to:references:date:message-id:subject:from:to
         :content-type;
        b=KKPoWCd5XabseVuwrU2LsK0eBoRVRVFv0OCfSOl6VL9N1KXj6ZHHkJ/WBE93aIOVvr
         vX19AimTqe5ED/vdBxZQ3FinDVSzl8Tf5X5T+ATPJjao23AE6ACnWwFaT4ql2ubiwoRT
         yrPzXGmRbKarjSaEUpL/Lx3JnyRxSeJN+1cek=
MIME-Version: 1.0
In-Reply-To: <B804A1908C36B543B6197825B8A9D43805AF12CA@sv-emis01.purple.local>
References: <B804A1908C36B543B6197825B8A9D43805AF12CA@sv-emis01.purple.local>
Date: Mon, 10 Jan 2011 15:31:54 +0900
Message-ID: <AANLkTinXeC+9xfOhzVJHYhDCOnMVj7L74q2XCO0PW6d1@mail.gmail.com>
Subject: Re: Connecting to Hadoop from Windows
From: Tost <ncanis@gmail.com>
To: user@hbase.apache.org
Content-Type: multipart/alternative; boundary=0016364eeacab9457e0499781f64

--0016364eeacab9457e0499781f64
Content-Type: text/plain; charset=ISO-8859-1

hadoop client library. see like below


======================
package com.hadoop;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * Hadoop File upload test
 * (If you want to upload big file, don't use hbase. just use hadoop
directly.)
 * @author ncanis@gmail.com
 *
 */
public class HadoopTest {
private Configuration conf = null;
public HadoopTest(){
conf = new Configuration();
}
 private FileSystem getDFS() throws IOException {
return FileSystem.get(conf);
}
 public void close(OutputStream os){
if(os==null) return;
try { os.close(); } catch (IOException e) {}
}
public void close(InputStream is){
if(is==null) return;
try { is.close(); } catch (IOException e) {}
}
 public boolean uploadFile(Path directory,String filename, InputStream is,
boolean overwrite){
boolean isSuccess=false;
FSDataOutputStream fos=null;
try {
Path p  = new Path(directory, new Path(filename));
FileSystem fs = getDFS();
if(fs.getFileStatus(directory).isDir()==false) {
throw new IOException(directory+" isn't directory.");
}else if(fs.exists(p)){
if(overwrite) {
delete(p,true);
}else{
throw new IOException(p+" already exist.");
}
}
 fos = fs.create(p);
BufferedInputStream bis = new BufferedInputStream(is);
// IOUtils.copyBytes(bis,fos,8192,true);
copyBytes(bis,fos,8192,true);
 isSuccess= true;
} catch (IOException e) {
e.printStackTrace();
} finally{
close(fos);
}
return isSuccess;
}
 /**
 * copy bytes.
 * you should use IOUtils.copyBytes(bis,fos,8192,true)
 * @param is
 * @param os
 * @param bufferSize
 * @param isLog
 * @throws IOException
 */
private void copyBytes(InputStream is, OutputStream os, int bufferSize,
boolean isLog) throws IOException{
int length=0;
long current = 0;
long total = is.available();
long progress = 0;
byte[] buf = new byte[bufferSize];
while((length=is.read(buf))!=(-1)){
os.write(buf,0, length);
current+=length;
long now = current*100/total;
if(progress!=now) {
log("progress=",now,"%");
progress = now;
}
}
}
 public void delete(Path p, boolean recursive) throws IOException{
FileSystem fs = getDFS();
fs.delete(p, recursive);
}
 public static final void log(Object ... os){
StringBuilder sb = new StringBuilder();
for(Object s:os){
sb.append(s);
}
System.out.println(sb.toString());
}
 /**
 * after use, must close inputstream.
 * @param p
 * @param bufferSize
 * @return
 * @throws IOException
 */
public InputStream path2Stream(Path p, int bufferSize) throws IOException{
FileSystem fs = getDFS();
FSDataInputStream fis = fs.open(p, bufferSize);
return fis;
}
 public boolean copyFile2Local(Path path, File f){
boolean isSuccess=false;
try {
InputStream is = path2Stream(path,8192);
write2File(is,f);
isSuccess= true;
} catch (IOException e) {
e.printStackTrace();
}
return isSuccess;
 }
/**
 * load file stream in hbase, write to file
 * @param is
 * @param f
 * @return
 */
public boolean write2File(InputStream is, File f){
boolean isSuccess=false;
FileOutputStream fos=null;
try {
 BufferedInputStream bis = new BufferedInputStream(is);
fos = new FileOutputStream(f);
// IOUtils.copyBytes(is,fos,8192,true);
copyBytes(bis,fos,8192,true);
isSuccess = true;
} catch (IOException e) {
e.printStackTrace();
} finally{
close(is);
close(fos);
}
return isSuccess;
}
 long start = 0L;
private void start(){
start = System.currentTimeMillis();
}
 private long end(){
return System.currentTimeMillis()-start;
}
/**
 * @param args
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
HadoopTest ht = new HadoopTest();
 File file = new File("doc/movie.avi");
FileInputStream fis = new FileInputStream(file);
Path rootPath = new Path("/files/");
String filename = "movie.avi";
 ht.start();
boolean isUploaded = ht.uploadFile(rootPath,filename,fis,true);
log("uploaded=",isUploaded," size=",file.length()," bytes. time=",ht.end(),"
ms");
 ht.start();
ht.copyFile2Local(new Path(rootPath,filename), new File("test/"+filename));
log("saved file time=",ht.end());
}

}


2011/1/10 Stuart Scott <Stuart.Scott@e-mis.com>

> Hi,
>
>
>
> Can anyone offer any advice please?
>
> We are successfully running a Hadoop cluster (Hbase/Hive etc..) but need
> to access this from a Windows operating system (for various legacy
> reasons!). We need to be able to put files/gets in/out of HDFS and also
> access HBase.
>
>
>
> Has anyone achieved this?
>
> Am I on to a none-starter?
>
>
>
> I have managed to build a Java application using the latest
> hadoop-core-0.20.2+320.jar's etc.. this compiles/runs but gives various
> errors when trying to reference Hadoop.
>
> e.g. Opening an InputStream Exception in thread "main"
> java.net.MalformedURLException: unknown protocol: hdfs
>
>
>
> Would we need the full Hadoop on the Client (which isn't part of the
> cluster)?
>
>
>
> Will this actually work?
>
>
>
> Any advice would be gratefully received.
>
>
>
> Regards
>
>
>
> Stuart Scott
>
> System Architect
> emis intellectual technology
> Fulford Grange, Micklefield Lane
> Rawdon Leeds LS19 6BA
> E-mail: stuart.scott@e-mis.com <mailto:stuart.scott@e-mis.com>
> Website: www.emisit.com <outbind://26/www.emisit.com>
>
> Privileged and/or Confidential information may be contained in this
> message. If you are not the original addressee indicated in this message
> (or responsible for delivery of the message to such person), you may not
> copy or deliver this message to anyone. In such case, please delete this
> message, and notify us immediately. Opinions, conclusions and other
> information expressed in this message are not given or endorsed by EMIS
> nor can I conclude contracts on its behalf unless otherwise indicated by
> an authorised representative independently of this message.
>
> EMIS reserves the right to monitor, intercept and (where appropriate)
> read all incoming and outgoing communications. By replying to this
> message and where necessary you are taken as being aware of and giving
> consent to such monitoring, interception and reading.
>
>
> EMIS is a trading name of Egton Medical Information Systems Limited.
> Registered in England. No 2117205. Registered Office: Fulford Grange,
> Micklefield Lane, Rawdon, Leeds, LS19 6BA
>
>
>
>

--0016364eeacab9457e0499781f64--