Return-Path: Delivered-To: apmail-hadoop-core-commits-archive@www.apache.org Received: (qmail 69819 invoked from network); 28 Feb 2008 14:44:27 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 28 Feb 2008 14:44:27 -0000 Received: (qmail 24437 invoked by uid 500); 28 Feb 2008 14:44:22 -0000 Delivered-To: apmail-hadoop-core-commits-archive@hadoop.apache.org Received: (qmail 24421 invoked by uid 500); 28 Feb 2008 14:44:22 -0000 Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: core-dev@hadoop.apache.org Delivered-To: mailing list core-commits@hadoop.apache.org Received: (qmail 24412 invoked by uid 99); 28 Feb 2008 14:44:22 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Feb 2008 06:44:22 -0800 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Feb 2008 14:43:56 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 4F39D1A9832; Thu, 28 Feb 2008 06:44:05 -0800 (PST) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r631986 - in /hadoop/core/trunk: ./ src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/fs/ src/test/org/apache/hadoop/dfs/ Date: Thu, 28 Feb 2008 14:44:03 -0000 To: core-commits@hadoop.apache.org From: dhruba@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080228144405.4F39D1A9832@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: dhruba Date: Thu Feb 28 06:43:44 2008 New Revision: 631986 URL: http://svn.apache.org/viewvc?rev=631986&view=rev Log: HADOOP-2063. A new parameter to dfs -get command to fetch a file even if it is corrupted. (Tsz Wo (Nicholas), SZE via dhruba) Modified: hadoop/core/trunk/CHANGES.txt hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java hadoop/core/trunk/src/java/org/apache/hadoop/fs/FSInputChecker.java hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java Modified: hadoop/core/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=631986&r1=631985&r2=631986&view=diff ============================================================================== --- hadoop/core/trunk/CHANGES.txt (original) +++ hadoop/core/trunk/CHANGES.txt Thu Feb 28 06:43:44 2008 @@ -26,6 +26,9 @@ HADOOP-2178. Job History on DFS. (Amareshwari Sri Ramadasu via ddas) + HADOOP-2063. A new parameter to dfs -get command to fetch a file + even if it is corrupted. (Tsz Wo (Nicholas), SZE via dhruba) + IMPROVEMENTS HADOOP-2655. Copy on write for data and metadata files in the Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?rev=631986&r1=631985&r2=631986&view=diff ============================================================================== --- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original) +++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Thu Feb 28 06:43:44 2008 @@ -252,7 +252,7 @@ } public DFSInputStream open(String src) throws IOException { - return open(src, conf.getInt("io.file.buffer.size", 4096)); + return open(src, conf.getInt("io.file.buffer.size", 4096), true); } /** * Create an input stream that obtains a nodelist from the @@ -260,10 +260,11 @@ * inner subclass of InputStream that does the right out-of-band * work. */ - public DFSInputStream open(String src, int buffersize) throws IOException { + DFSInputStream open(String src, int buffersize, boolean verifyChecksum + ) throws IOException { checkOpen(); // Get block info from namenode - return new DFSInputStream(src, buffersize); + return new DFSInputStream(src, buffersize, verifyChecksum); } /** @@ -773,10 +774,11 @@ } private BlockReader( String file, long blockId, DataInputStream in, - DataChecksum checksum, long startOffset, - long firstChunkOffset ) { + DataChecksum checksum, boolean verifyChecksum, + long startOffset, long firstChunkOffset ) { super(new Path("/blk_" + blockId + ":of:" + file)/*too non path-like?*/, - 1, (checksum.getChecksumSize() > 0) ? checksum : null, + 1, verifyChecksum, + checksum.getChecksumSize() > 0? checksum : null, checksum.getBytesPerChecksum(), checksum.getChecksumSize()); @@ -792,12 +794,17 @@ checksumSize = this.checksum.getChecksumSize(); } + static BlockReader newBlockReader(Socket sock, String file, long blockId, + long startOffset, long len, int bufferSize) throws IOException { + return newBlockReader(sock, file, blockId, startOffset, len, bufferSize, + true); + } + /** Java Doc required */ static BlockReader newBlockReader( Socket sock, String file, long blockId, long startOffset, long len, - int bufferSize) + int bufferSize, boolean verifyChecksum) throws IOException { - // in and out will be closed when sock is closed (by the caller) DataOutputStream out = new DataOutputStream( new BufferedOutputStream(sock.getOutputStream())); @@ -835,7 +842,7 @@ startOffset + " for file " + file); } - return new BlockReader( file, blockId, in, checksum, + return new BlockReader( file, blockId, in, checksum, verifyChecksum, startOffset, firstChunkOffset ); } @@ -882,7 +889,8 @@ private String src; private long prefetchSize = 10 * defaultBlockSize; - private BlockReader blockReader; + private BlockReader blockReader = null; + private boolean verifyChecksum; private LocatedBlocks locatedBlocks = null; private DatanodeInfo currentNode = null; private Block currentBlock = null; @@ -900,14 +908,13 @@ deadNodes.put(dnInfo, dnInfo); } - /** - */ - public DFSInputStream(String src, int buffersize) throws IOException { + DFSInputStream(String src, int buffersize, boolean verifyChecksum + ) throws IOException { + this.verifyChecksum = verifyChecksum; this.buffersize = buffersize; this.src = src; prefetchSize = conf.getLong("dfs.read.prefetch.size", prefetchSize); openInfo(); - blockReader = null; } /** @@ -1065,10 +1072,8 @@ Block blk = targetBlock.getBlock(); blockReader = BlockReader.newBlockReader(s, src, blk.getBlockId(), - offsetIntoBlock, - (blk.getNumBytes() - - offsetIntoBlock), - buffersize); + offsetIntoBlock, blk.getNumBytes() - offsetIntoBlock, + buffersize, verifyChecksum); return chosenNode; } catch (IOException ex) { // Put chosen node into dead list, continue @@ -1248,7 +1253,7 @@ BlockReader reader = BlockReader.newBlockReader(dn, src, block.getBlock().getBlockId(), - start, len, buffersize); + start, len, buffersize, verifyChecksum); int nread = reader.readAll(buf, offset, len); if (nread != len) { throw new IOException("truncated return from reader.read(): " + Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?rev=631986&r1=631985&r2=631986&view=diff ============================================================================== --- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java (original) +++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java Thu Feb 28 06:43:44 2008 @@ -40,6 +40,7 @@ private URI uri; DFSClient dfs; + private boolean verifyChecksum = true; public DistributedFileSystem() { } @@ -111,9 +112,14 @@ return dfs.getHints(getPathName(f), start, len); } + public void setVerifyChecksum(boolean verifyChecksum) { + this.verifyChecksum = verifyChecksum; + } + public FSDataInputStream open(Path f, int bufferSize) throws IOException { try { - return new DFSClient.DFSDataInputStream(dfs.open(getPathName(f),bufferSize)); + return new DFSClient.DFSDataInputStream( + dfs.open(getPathName(f), bufferSize, verifyChecksum)); } catch(RemoteException e) { if (IOException.class.getName().equals(e.getClassName()) && e.getMessage().startsWith( Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FSInputChecker.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FSInputChecker.java?rev=631986&r1=631985&r2=631986&view=diff ============================================================================== --- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FSInputChecker.java (original) +++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FSInputChecker.java Thu Feb 28 06:43:44 2008 @@ -37,6 +37,7 @@ /** The file name from which data is read from */ protected Path file; private Checksum sum; + private boolean verifyChecksum; private byte[] buf; private byte[] checksum; private int pos; @@ -66,8 +67,9 @@ * @param checksumSize the number byte of each checksum */ protected FSInputChecker( Path file, int numOfRetries, - Checksum sum, int chunkSize, int checksumSize ) { + boolean verifyChecksum, Checksum sum, int chunkSize, int checksumSize ) { this(file, numOfRetries); + this.verifyChecksum = verifyChecksum; set(sum, chunkSize, checksumSize); } @@ -93,7 +95,7 @@ /** Return true if there is a need for checksum verification */ protected synchronized boolean needChecksum() { - return sum != null; + return verifyChecksum && sum != null; } /** @@ -163,7 +165,7 @@ } } - /* + /** * Fills the buffer with a chunk data. * No mark is supported. * This method assumes that all data in the buffer has already been read in, Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java?rev=631986&r1=631985&r2=631986&view=diff ============================================================================== --- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java (original) +++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java Thu Feb 28 06:43:44 2008 @@ -39,9 +39,9 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.dfs.DistributedFileSystem; /** Provide command line access to a FileSystem. */ public class FsShell extends Configured implements Tool { @@ -56,6 +56,9 @@ modifFmt.setTimeZone(TimeZone.getTimeZone("UTC")); } static final String SETREP_SHORT_USAGE="-setrep [-R] [-w] "; + static final String GET_SHORT_USAGE = "-get [-ignoreCrc] [-crc] "; + static final String COPYTOLOCAL_SHORT_USAGE = GET_SHORT_USAGE.replace( + "-get", "-copyToLocal"); static final String TAIL_USAGE="-tail [-f] "; private static final DecimalFormat decimalFormat; static { @@ -142,7 +145,7 @@ /** * Obtain the indicated files that match the file pattern srcf * and copy them to the local name. srcf is kept. - * When copying mutiple files, the destination must be a directory. + * When copying multiple files, the destination must be a directory. * Otherwise, IOException is thrown. * @param argv: arguments * @param pos: Ignore everything before argv[pos] @@ -150,40 +153,59 @@ * @see org.apache.hadoop.fs.FileSystem.globPaths */ void copyToLocal(String[]argv, int pos) throws IOException { - if (argv.length-pos<2 || (argv.length-pos==2 && argv[pos].equalsIgnoreCase("-crc"))) { - System.err.println("Usage: -get [-crc] "); - throw new RuntimeException("Usage: -get [-crc] "); - } - boolean copyCrc = false; - if ("-crc".equalsIgnoreCase(argv[pos])) { - pos++; - copyCrc = true; - } - String srcf = argv[pos++]; - String dstf = argv[pos++]; - if (dstf.equals("-")) { + CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc"); + + String srcstr = null; + String dststr = null; + try { + List parameters = cf.parse(argv, pos); + srcstr = parameters.get(0); + dststr = parameters.get(1); + } + catch(IllegalArgumentException iae) { + System.err.println("Usage: java FsShell " + GET_SHORT_USAGE); + throw iae; + } + final boolean copyCrc = cf.options.get("crc"); + final boolean verifyChecksum = !cf.options.get("ignoreCrc"); + + if (dststr.equals("-")) { if (copyCrc) { System.err.println("-crc option is not valid when destination is stdout."); } - cat(srcf); + cat(srcstr, verifyChecksum); } else { - File dst = new File(dstf); - Path src = new Path(srcf); - FileSystem srcFs = src.getFileSystem(getConf()); - Path [] srcs = srcFs.globPaths(src); + File dst = new File(dststr); + Path srcpath = new Path(srcstr); + FileSystem srcFS = getSrcFileSystem(srcpath, verifyChecksum); + FileStatus[] srcs = srcFS.globStatus(srcpath); boolean dstIsDir = dst.isDirectory(); if (srcs.length > 1 && !dstIsDir) { throw new IOException("When copying multiple files, " + "destination should be a directory."); } - for (Path srcPath : srcs) { - File dstFile = (dstIsDir ? new File(dst, srcPath.getName()) : dst); - copyToLocal(srcFs, srcPath, dstFile, copyCrc); + for (FileStatus status : srcs) { + Path p = status.getPath(); + File f = dstIsDir? new File(dst, p.getName()): dst; + copyToLocal(srcFS, p, f, copyCrc); } } } /** + * Return the {@link FileSystem} specified by src and the conf. + * It the {@link FileSystem} supports checksum, set verifyChecksum. + */ + private FileSystem getSrcFileSystem(Path src, boolean verifyChecksum + ) throws IOException { + FileSystem srcFs = src.getFileSystem(getConf()); + if (srcFs instanceof DistributedFileSystem) { + ((DistributedFileSystem)srcFs).setVerifyChecksum(verifyChecksum); + } + return srcFs; + } + + /** * The prefix for the tmp file used in copyToLocal. * It must be at least three characters long, required by * {@link java.io.File#createTempFile(String, String, File)}. @@ -207,7 +229,7 @@ * copyCrc and useTmpFile (may be useTmpFile need not be an option). */ - if (!srcFS.isDirectory(src)) { + if (!srcFS.getFileStatus(src).isDir()) { if (dst.exists()) { // match the error message in FileUtil.checkDest(): throw new IOException("Target " + dst + " already exists"); @@ -298,7 +320,7 @@ * @exception: IOException * @see org.apache.hadoop.fs.FileSystem.globPaths */ - void cat(String srcf) throws IOException { + void cat(String src, boolean verifyChecksum) throws IOException { //cat behavior in Linux // [~/1207]$ ls ?.txt // x.txt z.txt @@ -307,6 +329,7 @@ // cat: y.txt: No such file or directory // zzz + Path srcPattern = new Path(src); new DelayedExceptionThrowing() { @Override void process(Path p, FileSystem srcFs) throws IOException { @@ -315,7 +338,7 @@ } printToStdout(srcFs.open(p)); } - }.process(srcf); + }.process(srcPattern, getSrcFileSystem(srcPattern, verifyChecksum)); } private class TextRecordInputStream extends InputStream { @@ -374,6 +397,7 @@ } void text(String srcf) throws IOException { + Path srcPattern = new Path(srcf); new DelayedExceptionThrowing() { @Override void process(Path p, FileSystem srcFs) throws IOException { @@ -382,7 +406,7 @@ } printToStdout(forMagic(p, srcFs)); } - }.process(srcf); + }.process(srcPattern, srcPattern.getFileSystem(getConf())); } /** @@ -1002,12 +1026,13 @@ // [~/1207]$ rm x.txt y.txt z.txt // rm: cannot remove `y.txt': No such file or directory + Path srcPattern = new Path(srcf); new DelayedExceptionThrowing() { @Override void process(Path p, FileSystem srcFs) throws IOException { delete(p, srcFs, recursive); } - }.process(srcf); + }.process(srcPattern, srcPattern.getFileSystem(getConf())); } /* delete a file */ @@ -1222,9 +1247,9 @@ "[-D ] [-ls ] [-lsr ] [-du ]\n\t" + "[-dus ] [-mv ] [-cp ] [-rm ]\n\t" + "[-rmr ] [-put ] [-copyFromLocal ]\n\t" + - "[-moveFromLocal ] [-get ]\n\t" + + "[-moveFromLocal ] [" + GET_SHORT_USAGE + "\n\t" + "[-getmerge [addnl]] [-cat ]\n\t" + - "[-copyToLocal ] [-moveToLocal ]\n\t" + + "[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal ]\n\t" + "[-mkdir ] [-report] [" + SETREP_SHORT_USAGE + "]\n\t" + "[-touchz ] [-test -[ezd] ] [-stat [format] ]\n\t" + "[-tail [-f] ] [-text ]\n\t" + @@ -1298,7 +1323,8 @@ String moveFromLocal = "-moveFromLocal : Same as -put, except that the source is\n" + "\t\tdeleted after it's copied.\n"; - String get = "-get : Copy files that match the file pattern \n" + + String get = GET_SHORT_USAGE + + ": Copy files that match the file pattern \n" + "\t\tto the local name. is kept. When copying mutiple, \n" + "\t\tfiles, the destination must be a directory. \n"; @@ -1313,7 +1339,8 @@ "\t\tmatch a magic number associated with a known format\n" + "\t\t(gzip, SequenceFile)\n"; - String copyToLocal = "-copyToLocal : Identical to the -get command.\n"; + String copyToLocal = COPYTOLOCAL_SHORT_USAGE + + ": Identical to the -get command.\n"; String moveToLocal = "-moveToLocal : Not implemented yet \n"; @@ -1473,7 +1500,7 @@ // issue the command to the fs // if ("-cat".equals(cmd)) { - cat(argv[i]); + cat(argv[i], true); } else if ("-mkdir".equals(cmd)) { mkdir(argv[i]); } else if ("-rm".equals(cmd)) { @@ -1496,7 +1523,7 @@ } catch (RemoteException e) { // // This is a error returned by hadoop server. Print - // out the first line of the error mesage. + // out the first line of the error message. // exitCode = -1; try { @@ -1548,16 +1575,16 @@ "-moveFromLocal".equals(cmd)) { System.err.println("Usage: java FsShell" + " [" + cmd + " ]"); - } else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd) || - "-moveToLocal".equals(cmd)) { + } else if ("-get".equals(cmd)) { + System.err.println("Usage: java FsShell [" + GET_SHORT_USAGE + "]"); + } else if ("-copyToLocal".equals(cmd)) { + System.err.println("Usage: java FsShell [" + COPYTOLOCAL_SHORT_USAGE+ "]"); + } else if ("-moveToLocal".equals(cmd)) { System.err.println("Usage: java FsShell" + " [" + cmd + " [-crc] ]"); } else if ("-cat".equals(cmd)) { System.err.println("Usage: java FsShell" + " [" + cmd + " ]"); - } else if ("-get".equals(cmd)) { - System.err.println("Usage: java FsShell" + - " [" + cmd + " [addnl]]"); } else if ("-setrep".equals(cmd)) { System.err.println("Usage: java FsShell [" + SETREP_SHORT_USAGE + "]"); } else if ("-test".equals(cmd)) { @@ -1582,11 +1609,11 @@ System.err.println(" [-put ]"); System.err.println(" [-copyFromLocal ]"); System.err.println(" [-moveFromLocal ]"); - System.err.println(" [-get [-crc] ]"); + System.err.println(" [" + GET_SHORT_USAGE + "]"); System.err.println(" [-getmerge [addnl]]"); System.err.println(" [-cat ]"); System.err.println(" [-text ]"); - System.err.println(" [-copyToLocal [-crc] ]"); + System.err.println(" [" + COPYTOLOCAL_SHORT_USAGE + "]"); System.err.println(" [-moveToLocal [-crc] ]"); System.err.println(" [-mkdir ]"); System.err.println(" [" + SETREP_SHORT_USAGE + "]"); @@ -1793,17 +1820,16 @@ System.exit(res); } - /* + /** * Accumulate exceptions if there is any. Throw them at last. */ private abstract class DelayedExceptionThrowing { abstract void process(Path p, FileSystem srcFs) throws IOException; - void process(String srcf) throws IOException { + final void processSrc(Path srcPattern, FileSystem srcFs + ) throws IOException { List exceptions = new ArrayList(); - Path srcPath = new Path(srcf); - FileSystem srcFs = srcPath.getFileSystem(getConf()); - for(Path p : srcFs.globPaths(new Path(srcf))) + for(Path p : srcFs.globPaths(srcPattern)) try { process(p, srcFs); } catch(IOException ioe) { exceptions.add(ioe); } Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java?rev=631986&r1=631985&r2=631986&view=diff ============================================================================== --- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java (original) +++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java Thu Feb 28 06:43:44 2008 @@ -18,8 +18,10 @@ package org.apache.hadoop.dfs; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; import java.io.IOException; -import java.net.URI; import java.util.Random; import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; @@ -37,7 +39,6 @@ private static String[] dirNames = { "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine" }; - private static Configuration conf = new Configuration(); private int maxLevels;// = 3; private int maxSize;// = 8*1024; @@ -220,4 +221,17 @@ in.readByte(); return in.getCurrentBlock(); } + + static void setLogLevel2All(org.apache.commons.logging.Log log) { + ((org.apache.commons.logging.impl.Log4JLogger)log + ).getLogger().setLevel(org.apache.log4j.Level.ALL); + } + + static String readFile(File f) throws IOException { + StringBuilder b = new StringBuilder(); + BufferedReader in = new BufferedReader(new FileReader(f)); + for(int c; (c = in.read()) != -1; b.append((char)c)); + in.close(); + return b.toString(); + } } Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java?rev=631986&r1=631985&r2=631986&view=diff ============================================================================== --- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java (original) +++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java Thu Feb 28 06:43:44 2008 @@ -53,8 +53,9 @@ static File createLocalFile(File f) throws IOException { assertTrue(!f.exists()); - PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(f))); - out.println(f.getAbsolutePath()); + PrintWriter out = new PrintWriter(f); + out.print("createLocalFile: " + f.getAbsolutePath()); + out.flush(); out.close(); assertTrue(f.exists()); assertTrue(f.isFile()); @@ -863,6 +864,94 @@ fileSys.close(); } catch (Exception e) { } + cluster.shutdown(); + } + } + + static List getBlockFiles(MiniDFSCluster cluster) throws IOException { + List files = new ArrayList(); + List datanodes = cluster.getDataNodes(); + Block[][] blocks = cluster.getAllBlockReports(); + for(int i = 0; i < blocks.length; i++) { + FSDataset ds = (FSDataset)datanodes.get(i).getFSDataset(); + for(Block b : blocks[i]) { + files.add(ds.getBlockFile(b)); + } + } + return files; + } + + static void corrupt(List files) throws IOException { + for(File f : files) { + StringBuilder content = new StringBuilder(DFSTestUtil.readFile(f)); + char c = content.charAt(0); + content.setCharAt(0, ++c); + PrintWriter out = new PrintWriter(f); + out.print(content); + out.flush(); + out.close(); + } + } + + static interface TestGetRunner { + String run(int exitcode, String... options) throws IOException; + } + + public void testGet() throws IOException { + DFSTestUtil.setLogLevel2All(FSInputChecker.LOG); + final Configuration conf = new Configuration(); + MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); + DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem(); + + try { + final String fname = "testGet.txt"; + final File localf = createLocalFile(new File(TEST_ROOT_DIR, fname)); + final String localfcontent = DFSTestUtil.readFile(localf); + final Path root = mkdir(dfs, new Path("/test/get")); + final Path remotef = new Path(root, fname); + dfs.copyFromLocalFile(false, false, new Path(localf.getPath()), remotef); + + final FsShell shell = new FsShell(); + shell.setConf(conf); + TestGetRunner runner = new TestGetRunner() { + private int count = 0; + + public String run(int exitcode, String... options) throws IOException { + String dst = TEST_ROOT_DIR + "/" + fname+ ++count; + String[] args = new String[options.length + 3]; + args[0] = "-get"; + args[args.length - 2] = remotef.toString(); + args[args.length - 1] = dst; + for(int i = 0; i < options.length; i++) { + args[i + 1] = options[i]; + } + show("args=" + Arrays.asList(args)); + + try { + assertEquals(exitcode, shell.run(args)); + } catch (Exception e) { + assertTrue(StringUtils.stringifyException(e), false); + } + return exitcode == 0? DFSTestUtil.readFile(new File(dst)): null; + } + }; + + assertEquals(localfcontent, runner.run(0)); + assertEquals(localfcontent, runner.run(0, "-ignoreCrc")); + + //find and modify the block files + List files = getBlockFiles(cluster); + show("files=" + files); + corrupt(files); + + assertEquals(null, runner.run(-1)); + String corruptedcontent = runner.run(0, "-ignoreCrc"); + assertEquals(localfcontent.substring(1), corruptedcontent.substring(1)); + assertEquals(localfcontent.charAt(0)+1, corruptedcontent.charAt(0)); + + localf.delete(); + } finally { + try {dfs.close();} catch (Exception e) {} cluster.shutdown(); } }