Return-Path: X-Original-To: apmail-hadoop-user-archive@minotaur.apache.org Delivered-To: apmail-hadoop-user-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CFB3710E85 for ; Fri, 16 Aug 2013 06:33:45 +0000 (UTC) Received: (qmail 18770 invoked by uid 500); 16 Aug 2013 06:33:38 -0000 Delivered-To: apmail-hadoop-user-archive@hadoop.apache.org Received: (qmail 18621 invoked by uid 500); 16 Aug 2013 06:33:33 -0000 Mailing-List: contact user-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: user@hadoop.apache.org Delivered-To: mailing list user@hadoop.apache.org Received: (qmail 18608 invoked by uid 99); 16 Aug 2013 06:33:31 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 16 Aug 2013 06:33:31 +0000 X-ASF-Spam-Status: No, hits=1.5 required=5.0 tests=HTML_MESSAGE,RCVD_IN_DNSWL_LOW,SPF_PASS X-Spam-Check-By: apache.org Received-SPF: pass (athena.apache.org: domain of justlooks@gmail.com designates 209.85.219.53 as permitted sender) Received: from [209.85.219.53] (HELO mail-oa0-f53.google.com) (209.85.219.53) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 16 Aug 2013 06:33:26 +0000 Received: by mail-oa0-f53.google.com with SMTP id k18so1893481oag.12 for ; Thu, 15 Aug 2013 23:33:05 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113; h=mime-version:date:message-id:subject:from:to:content-type; bh=GJqC+bc7gtV8fWZdVlqtUIxSz44smZCvkaq2oHSdtXs=; b=o93NklEWK0NOAEqUzwYABtzUxCYSOablRpgfgSFi59ZrGKoHy8Q7WzdUtKNO9aN9Q8 SlbNU9AEm/jgIF8HMYKfz7fr0xIJjQYA++cHfhoGBiIc5Sscczo/Et4sli67yakikBBE EZ6OsGrSGccbihWBWMJLAVsrkUrUiaFgWF0GivuuU2cMrJ/oskuJM/eMqDAVQKOVOaG9 TqYFTvQP23iBX/OwOMwT2nVwOsSXhqQeBJx8w9KSODabY92TPITI8N10j9L5IbuPBtUt Xl+v2cJR1OZZTn9pax1G+LlzYqydGcWw80d+KY9NNgFkmrOwYoVys7JtP+w6kZGYj4Pi QENg== MIME-Version: 1.0 X-Received: by 10.60.131.232 with SMTP id op8mr42364oeb.75.1376634785797; Thu, 15 Aug 2013 23:33:05 -0700 (PDT) Received: by 10.182.129.138 with HTTP; Thu, 15 Aug 2013 23:33:05 -0700 (PDT) Date: Fri, 16 Aug 2013 14:33:05 +0800 Message-ID: Subject: " No FileSystem for scheme: hdfs " in namenode HA From: ch huang To: user@hadoop.apache.org Content-Type: multipart/alternative; boundary=047d7b47266a649e9404e40ac354 X-Virus-Checked: Checked by ClamAV on apache.org --047d7b47266a649e9404e40ac354 Content-Type: text/plain; charset=ISO-8859-1 hi,all i setup namenode HA hadoop cluster and write some demo code import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class TestConnect { private static void appendToHdfs(String content,String dst) throws FileNotFoundException,IOException { Configuration conf = new Configuration(); conf.set("dfs.replication", "2"); // System.out.println("append is : "+conf.get("dfs.support.append")); // System.out.println("append is : "+conf.get("dfs.name.dir")); FileSystem fs = FileSystem.get(URI.create(dst), conf); FSDataOutputStream out = fs.append(new Path(dst)); int readLen = content.getBytes().length; out.write(content.getBytes(), 0, readLen); out.close(); fs.close(); } public static void createNewHDFSFile(String toCreateFilePath, String content) throws IOException { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(URI.create(toCreateFilePath),config); FSDataOutputStream os = hdfs.create(new Path(toCreateFilePath)); os.write(content.getBytes("UTF-8")); os.close(); hdfs.close(); } public static void listAll(String dir) throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(dir),conf); FileStatus[] stats = fs.listStatus(new Path(dir)); for(int i = 0; i < stats.length; ++i) { if (stats[i].isFile()) { // regular file System.out.println(stats[i].getPath().toString()); } else if (stats[i].isDirectory()) { // dir System.out.println(stats[i].getPath().toString()); } else if(stats[i].isSymlink()) { // is s symlink in linux System.out.println(stats[i].getPath().toString()); } } fs.close(); } public static void main(String[] args) { // TODO Auto-generated method stub try { createNewHDFSFile("hdfs://mycluster/alex","mycluster"); listAll("hdfs://mycluster/alex"); Configuration config = new Configuration(); System.out.println("append is : "+config.get("dfs.hosts")); } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } and client configuration file :hdfs-site.xml fs.defaultFS hdfs://mycluster ha.zookeeper.quorum node1:2181,node2:2181,node3:2181 dfs.nameservices mycluster dfs.ha.namenodes.mycluster nn1,nn2 dfs.namenode.rpc-address.mycluster.nn1 node1:8020 dfs.namenode.rpc-address.mycluster.nn2 node2:8020 dfs.namenode.shared.edits.dir qjournal://node1:8485;node2:8485;node3:8485/mycluster dfs.client.failover.proxy.provider.mycluster org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider when i run the test ,i get some error information,any one can help? log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory). log4j:WARN Please initialize the log4j system properly. log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info. java.io.IOException: No FileSystem for scheme: hdfs at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2296) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2303) at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:87) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2342) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2324) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:351) at TestConnect.createNewHDFSFile(TestConnect.java:35) at TestConnect.main(TestConnect.java:80) --047d7b47266a649e9404e40ac354 Content-Type: text/html; charset=ISO-8859-1 Content-Transfer-Encoding: quoted-printable
hi,all i setup namenode HA hadoop cluster
=A0
and write some demo code
=A0
import java.io.FileNotFoundException;
import java.io.IOException;import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hado= op.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
imp= ort org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class TestConnect {
=A0private static void appendToHdfs(String content,String dst) throws = FileNotFoundException,IOException {
=A0=A0
=A0=A0=A0 Configuration co= nf =3D new Configuration();
=A0=A0=A0 conf.set("dfs.replication&quo= t;, "2");
=A0=A0//=A0 System.out.println("append is : "+conf.get("dfs.= support.append"));
=A0=A0//=A0 System.out.println("append is := "+conf.get("dfs.name.dir"));
=A0=A0=A0 FileSystem fs =3D= FileSystem.get(URI.create(dst), conf);=A0
=A0=A0=A0 FSDataOutputStream out =3D fs.append(new Path(dst));
=A0=A0=A0 int readLen =3D content.getBytes().length;
=A0=A0=A0
=A0=A0=A0 out.write(content.getBytes(), 0, readLen);
= =A0=A0=A0
=A0=A0=A0 out.close();
=A0=A0=A0 fs.close();
=A0=A0 }=A0
=A0=A0=A0 public static void createNewHDFSFile(String toCreateFile= Path, String content) throws IOException
=A0=A0=A0=A0 {
=A0=A0=A0=A0=A0=A0=A0=A0 Configuration config =3D new Con= figuration();
=A0=A0=A0=A0=A0=A0=A0=A0 FileSystem hdfs =3D FileSystem.ge= t(URI.create(toCreateFilePath),config);
=A0=A0=A0=A0=A0=A0=A0=A0
=A0= =A0=A0=A0=A0=A0=A0=A0 FSDataOutputStream os =3D hdfs.create(new Path(toCrea= teFilePath));
=A0=A0=A0=A0=A0=A0=A0=A0 os.write(content.getBytes("UTF-8"))= ;
=A0=A0=A0=A0=A0=A0=A0=A0
=A0=A0=A0=A0=A0=A0=A0=A0 os.close();
= =A0=A0=A0=A0=A0=A0=A0=A0
=A0=A0=A0=A0=A0=A0=A0=A0 hdfs.close();
=A0= =A0=A0=A0 }
=A0
=A0=A0=A0 public static void listAll(String dir) thro= ws IOException
=A0=A0=A0=A0 {
=A0=A0=A0=A0=A0=A0=A0=A0 Configuration conf =3D new Confi= guration();
=A0=A0=A0=A0=A0=A0=A0=A0 FileSystem fs =3D FileSystem.get(UR= I.create(dir),conf);
=A0=A0=A0=A0=A0=A0=A0=A0
=A0=A0=A0=A0=A0=A0=A0= =A0 FileStatus[] stats =3D fs.listStatus(new Path(dir));
=A0=A0=A0=A0=A0= =A0=A0=A0
=A0=A0=A0=A0=A0=A0=A0=A0 for(int i =3D 0; i < stats.length= ; ++i)
=A0=A0=A0=A0=A0=A0=A0=A0 {
=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0 if (stat= s[i].isFile())
=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0 {
=A0=A0=A0=A0=A0= =A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0 // regular file
=A0=A0=A0=A0=A0=A0=A0= =A0=A0=A0=A0=A0=A0=A0=A0=A0 System.out.println(stats[i].getPath().toString(= ));
=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0 }
=A0=A0=A0=A0=A0=A0=A0=A0= =A0=A0=A0=A0 else if (stats[i].isDirectory())
=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0 {
=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0= =A0=A0=A0=A0=A0 // dir
=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0 = System.out.println(stats[i].getPath().toString());
=A0=A0=A0=A0=A0=A0=A0= =A0=A0=A0=A0=A0 }
=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0 else if(stats[i].= isSymlink())
=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0 {
=A0=A0=A0=A0=A0= =A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0 // is s symlink in linux
=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0 System.out.println(stats[i= ].getPath().toString());
=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0 }
=A0= =A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0=A0
=A0=A0=A0=A0=A0=A0=A0= =A0 }
=A0=A0=A0=A0=A0=A0=A0=A0 fs.close();
=A0=A0=A0=A0 }
=A0publi= c static void main(String[] args) {
=A0=A0
=A0=A0// TODO Auto-generated method stub
=A0=A0try {
=A0=A0=A0=A0=A0=A0createNewHDFSFile("hdfs://mycluster/alex","myclus= ter");
=A0=A0=A0listAll("hdfs://mycluster/alex");
=A0= =A0=A0Configuration config =3D new Configuration();
=A0=A0=A0System.out.println("append is : "+config.get("= dfs.hosts"));
=A0=A0} catch (FileNotFoundException e) {
=A0=A0= =A0// TODO Auto-generated catch block
=A0=A0=A0e.printStackTrace();
= =A0=A0} catch (IOException e) {
=A0=A0=A0// TODO Auto-generated catch block
=A0=A0=A0e.printStackTrace()= ;
=A0=A0}
=A0}
=A0
}
and client configuration file :hdfs-site.xml
=A0
<property>
=A0<name>fs.defaultFS</name>
=A0<= ;value>hdfs://mycluster</value>
</property>
<property>
=A0=A0=A0=A0=A0 <name>ha.zookeeper.quorum<= ;/name>
=A0=A0=A0=A0=A0 <value>node1:2181,node2:2181,node3:2181= </value>
</property>

<property>
=A0=A0=A0=A0=A0=A0=A0 <name>dfs.nameserv= ices</name>
=A0=A0=A0=A0=A0=A0=A0 <value>mycluster</value= >
</property>
<property>
=A0=A0=A0=A0=A0=A0=A0 <name>dfs.ha.namenodes= .mycluster</name>
=A0=A0=A0=A0=A0=A0=A0 <value>nn1,nn2</v= alue>
</property>
<property>
=A0=A0=A0=A0=A0=A0=A0 <name>dfs.namenode.rpc= -address.mycluster.nn1</name>
=A0=A0=A0=A0=A0=A0=A0 <value>n= ode1:8020</value>
</property>
<property>
=A0=A0= =A0=A0=A0=A0=A0 <name>dfs.namenode.rpc-address.mycluster.nn2</name= >
=A0=A0=A0=A0=A0=A0=A0 <value>node2:8020</value>
</propert= y>
<property>
=A0=A0=A0=A0=A0=A0=A0 <name>dfs.namenode.sha= red.edits.dir</name>
=A0=A0=A0=A0=A0=A0=A0 <value>qjournal:/= /node1:8485;node2:8485;node3:8485/mycluster</value>
</property&= gt;

<property>
=A0=A0=A0=A0=A0=A0=A0 <name>dfs.client.f= ailover.proxy.provider.mycluster</name>
=A0=A0=A0=A0=A0=A0=A0 <= value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyP= rovider</value>
</property>
=A0
when i run the test ,i get some error information,any one can help?
=A0
log4j:WARN No appenders could be found for logger (org.apache.hadoop.m= etrics2.lib.MutableMetricsFactory).
log4j:WARN Please initialize the log= 4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#n= oconfig for more info.
java.io.IOException: No FileSystem for scheme: hdfs
=A0at org.apache.had= oop.fs.FileSystem.getFileSystemClass(FileSystem.java:2296)
=A0at org.apa= che.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2303)
=A0at or= g.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:87)
=A0at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:234= 2)
=A0at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2324)=
=A0at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:351)
=A0at= TestConnect.createNewHDFSFile(TestConnect.java:35)
=A0at TestConnect.main(TestConnect.java:80)
--047d7b47266a649e9404e40ac354--