hadoop-common-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "11 Nov." <nov.eleve...@gmail.com>
Subject Can't get DFS file size
Date Tue, 08 Apr 2008 02:58:45 GMT
Hi folks,
    I'm writing a little test programm to check the writing speed of DFS
file system, but can't get the file size using
"fs.getFileStatus(file).getLen()" or fs.getContentLength(file). Here is my
code:

package org.apache.hadoop.fs;
import java.io.IOException;
import java.io.OutputStream;

import junit.framework.TestCase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Text;

public class TestDFSWrite extends TestCase {
  static String ROOT = System.getProperty("test.build.data","fs_test");
  static Path DATA_DIR = new Path(ROOT, "fs_data");
  static long MEGA = 1024 * 1024;
  static int BUFFER_SIZE = 4096;
  static Configuration conf = new Configuration();
  static FileSystem fs;

  static byte[] buffer = new byte[BUFFER_SIZE];
  static boolean finished;

  public class FileStatusChecker extends Thread{
    Path file;
    Path resultFile;
    int interval;

    public FileStatusChecker(Path file, Path resultFile, int interval){
        this.file = file;
        this.resultFile = resultFile;
        this.interval = interval;
      }

      public void run(){
        System.out.println("Here is the checker running!!");
        System.out.println(file.toString());
        SequenceFile.Writer writer = null;
        try {
          long lastLen = 1;
          long thisLen = 1;
        writer = SequenceFile.createWriter(fs, conf, resultFile,
                                      Text.class, Text.class,
CompressionType.NONE);

        while(!finished){
          lastLen = thisLen;
          if(fs.exists(file)){
            System.out.println("File exists!");
            thisLen = fs.getContentLength(file);
            System.out.println("@@"+thisLen);
          }
          else{
            sleep(interval * 10);
            continue;
          }
          long length = thisLen -lastLen;
          System.out.println("thisLen is" + thisLen + "lastLen is " +
lastLen);
          long cur = System.currentTimeMillis();
          cur = cur - (cur % 10);
          LongWritable current = new LongWritable(cur);
          writer.append(new Text(current.toString()), new Text(new
LongWritable(length).toString()));
          sleep(interval * 10);

        }
        return;

      }catch (Exception e) {
        e.printStackTrace();
      }finally {
          try {
              writer.close();
          } catch (Exception e) {
              e.printStackTrace();
          }
      }
    }
  }

  public static void main(String[] args) throws Exception {
    {
      try {
        fs = FileSystem.get(conf);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
    String testFunc = "";
    String fileName = "";
    String resultFileName = "";
    int fileSize = 0;
    int interval = 0;

    String usage = "Usage: TestDFSWrite -testfunc [read/write] -file foo
-size M -interval MS -result resultFile";
    if (args.length == 0) {
        System.err.println(usage);
        System.exit(-1);
      }

    for (int i = 0; i < args.length; i++) {       // parse command line
        if (args[i].equals("-testfunc")) {
          testFunc = args[++i];
        } else if (args[i].equals("-file")) {
          fileName = args[++i];
        } else if (args[i].equals("-size")) {
          fileSize = Integer.parseInt(args[++i]);
        } else if (args[i].equals("-interval")) {
          interval = Integer.parseInt(args[++i]);
        } else if (args[i].equals("-result")) {
          resultFileName = args[++i];
        }
      }

    long total = fileSize * MEGA;
    OutputStream out;
    Path file, resultFile;
    fs.delete(DATA_DIR);

    if(testFunc.equals("read")){
      System.out.println("This option is not ready.");
      return;
    }else if(testFunc.equals("write")){
      file = new Path(DATA_DIR, fileName);
      resultFile = new Path(DATA_DIR, resultFileName);
    }else{
      System.out.println("Invalid command line option.");
      return;
    }

    FileStatusChecker checker = new TestDFSWrite().new
FileStatusChecker(file, resultFile, interval);
    System.out.println(file.toString());
    //System.out.println("F L:" +
newLongWritable(fs.getContentLength(file)).toString());
    out = fs.create(file);
    checker.start();
    long written = 0;
    try {
      finished = false;
      while (written < total) {
        long remains = total - written;
        int length = (remains<=buffer.length) ? (int)remains :
buffer.length;
        out.write(buffer, 0, length);
        out.flush();
        written += length;
        System.out.println("One segment done!");
        System.out.println("F L:" + new
LongWritable(fs.getFileStatus(file).getLen()).toString());
      }
      finished =true;
    } finally {
      out.close();
    }
    //System.exit(0);
  }
}

Neither the inner class FileStatusChecker nor the main function can get the
right size, they both get "0".

What should be the problem?

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message