hadoop-common-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Michaela Buergle <Michaela.Buer...@neofonie.de>
Subject Re: Can't get DFS file size
Date Tue, 08 Apr 2008 10:13:29 GMT
Maybe because you pass Strings to the LongWritables?

micha

11 Nov. wrote:
> Hi folks,
>     I'm writing a little test programm to check the writing speed of DFS
> file system, but can't get the file size using
> "fs.getFileStatus(file).getLen()" or fs.getContentLength(file). Here is my
> code:
> 
> package org.apache.hadoop.fs;
> import java.io.IOException;
> import java.io.OutputStream;
> 
> import junit.framework.TestCase;
> 
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.fs.FileSystem;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.SequenceFile;
> import org.apache.hadoop.io.SequenceFile.CompressionType;
> import org.apache.hadoop.io.Text;
> 
> public class TestDFSWrite extends TestCase {
>   static String ROOT = System.getProperty("test.build.data","fs_test");
>   static Path DATA_DIR = new Path(ROOT, "fs_data");
>   static long MEGA = 1024 * 1024;
>   static int BUFFER_SIZE = 4096;
>   static Configuration conf = new Configuration();
>   static FileSystem fs;
> 
>   static byte[] buffer = new byte[BUFFER_SIZE];
>   static boolean finished;
> 
>   public class FileStatusChecker extends Thread{
>     Path file;
>     Path resultFile;
>     int interval;
> 
>     public FileStatusChecker(Path file, Path resultFile, int interval){
>         this.file = file;
>         this.resultFile = resultFile;
>         this.interval = interval;
>       }
> 
>       public void run(){
>         System.out.println("Here is the checker running!!");
>         System.out.println(file.toString());
>         SequenceFile.Writer writer = null;
>         try {
>           long lastLen = 1;
>           long thisLen = 1;
>         writer = SequenceFile.createWriter(fs, conf, resultFile,
>                                       Text.class, Text.class,
> CompressionType.NONE);
> 
>         while(!finished){
>           lastLen = thisLen;
>           if(fs.exists(file)){
>             System.out.println("File exists!");
>             thisLen = fs.getContentLength(file);
>             System.out.println("@@"+thisLen);
>           }
>           else{
>             sleep(interval * 10);
>             continue;
>           }
>           long length = thisLen -lastLen;
>           System.out.println("thisLen is" + thisLen + "lastLen is " +
> lastLen);
>           long cur = System.currentTimeMillis();
>           cur = cur - (cur % 10);
>           LongWritable current = new LongWritable(cur);
>           writer.append(new Text(current.toString()), new Text(new
> LongWritable(length).toString()));
>           sleep(interval * 10);
> 
>         }
>         return;
> 
>       }catch (Exception e) {
>         e.printStackTrace();
>       }finally {
>           try {
>               writer.close();
>           } catch (Exception e) {
>               e.printStackTrace();
>           }
>       }
>     }
>   }
> 
>   public static void main(String[] args) throws Exception {
>     {
>       try {
>         fs = FileSystem.get(conf);
>       } catch (IOException e) {
>         throw new RuntimeException(e);
>       }
>     }
>     String testFunc = "";
>     String fileName = "";
>     String resultFileName = "";
>     int fileSize = 0;
>     int interval = 0;
> 
>     String usage = "Usage: TestDFSWrite -testfunc [read/write] -file foo
> -size M -interval MS -result resultFile";
>     if (args.length == 0) {
>         System.err.println(usage);
>         System.exit(-1);
>       }
> 
>     for (int i = 0; i < args.length; i++) {       // parse command line
>         if (args[i].equals("-testfunc")) {
>           testFunc = args[++i];
>         } else if (args[i].equals("-file")) {
>           fileName = args[++i];
>         } else if (args[i].equals("-size")) {
>           fileSize = Integer.parseInt(args[++i]);
>         } else if (args[i].equals("-interval")) {
>           interval = Integer.parseInt(args[++i]);
>         } else if (args[i].equals("-result")) {
>           resultFileName = args[++i];
>         }
>       }
> 
>     long total = fileSize * MEGA;
>     OutputStream out;
>     Path file, resultFile;
>     fs.delete(DATA_DIR);
> 
>     if(testFunc.equals("read")){
>       System.out.println("This option is not ready.");
>       return;
>     }else if(testFunc.equals("write")){
>       file = new Path(DATA_DIR, fileName);
>       resultFile = new Path(DATA_DIR, resultFileName);
>     }else{
>       System.out.println("Invalid command line option.");
>       return;
>     }
> 
>     FileStatusChecker checker = new TestDFSWrite().new
> FileStatusChecker(file, resultFile, interval);
>     System.out.println(file.toString());
>     //System.out.println("F L:" +
> newLongWritable(fs.getContentLength(file)).toString());
>     out = fs.create(file);
>     checker.start();
>     long written = 0;
>     try {
>       finished = false;
>       while (written < total) {
>         long remains = total - written;
>         int length = (remains<=buffer.length) ? (int)remains :
> buffer.length;
>         out.write(buffer, 0, length);
>         out.flush();
>         written += length;
>         System.out.println("One segment done!");
>         System.out.println("F L:" + new
> LongWritable(fs.getFileStatus(file).getLen()).toString());
>       }
>       finished =true;
>     } finally {
>       out.close();
>     }
>     //System.exit(0);
>   }
> }
> 
> Neither the inner class FileStatusChecker nor the main function can get the
> right size, they both get "0".
> 
> What should be the problem?
> 

-- 
--------------------------------
Michaela B├╝rgle
Softwareentwicklerin
Forschung und Entwicklung

neofonie GmbH
Robert-Koch-Platz 4
10115 Berlin
fon: +49.30 24627 256
fax: +49.30 24627 120
michaela.buergle@neofonie.de
http://www.neofonie.de

Handelsregister
Berlin-Charlottenburg: HRB 67460

Geschaeftsfuehrung
Helmut Hoffer von Ankershoffen
Nurhan Yildirim
--------------------------------

Mime
View raw message