hadoop-mapreduce-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Niels Basjes <Ni...@basjes.nl>
Subject Re: Issue on accessing AVRO format file from HA Cluster.
Date Wed, 27 Apr 2016 18:46:12 GMT
Hi,

You say are on an HA cluster; yet by just looking at the errors I see the
stack being routed through "org.apache.hadoop.hdfs.NameNodeProxies.
createNonHAProxy"
My best guess is that you HA config is incomplete.

Niels Basjes


On Wed, Apr 27, 2016 at 4:27 PM, Mayank Mishra <mayank.mishra.nitb@gmail.com
> wrote:

> Hi,
>
> Kindly advise me to resolve the below exception getting while trying to
> access the AVRO file on HA cluster from the below code.  I have attached
> the resource file and pasted too.
>
>
> *Code: **import* java.io.IOException;
>
> *import* org.apache.avro.file.DataFileReader;
>
> *import* org.apache.avro.file.FileReader;
>
> *import* org.apache.avro.file.SeekableInput;
>
> *import* org.apache.avro.generic.GenericDatumReader;
>
> *import* org.apache.avro.generic.GenericRecord;
>
> *import* org.apache.avro.io.DatumReader;
>
> *import* org.apache.avro.mapred.FsInput;
>
> *import* org.apache.hadoop.conf.Configuration;
>
> *import* org.apache.hadoop.fs.Path;
>
> *public* *static* *void* main(String args[]) *throws* IOException{
>
>            Path path = *new* Path(
> "hdfs://infaqaha/user/hive/warehouse/avro_table_string/000000_0");
>
>            Configuration config = *new* Configuration(); // make this
> your *Hadoop* *env* *config*
>
>            config.addResource(
> "C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\hadoop-conf\\hdfs-site.xml"
> );
>
>            config.addResource(
> "C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\core-site");
>
>
>
>            //*Metadata* fetch for each record, we sample only one record
> in actual implementation.
>
>            SeekableInput input = *new* FsInput(path, config); <-Exception
> on this line
>
>            DatumReader<GenericRecord> reader = *new*
> GenericDatumReader<GenericRecord>();
>
>            FileReader<GenericRecord> fileReader = DataFileReader.
> *openReader*(input, reader);
>
> fileReader.close(); // also closes underlying FsInput
> }
>
>
>
>
> *Exception is : *
>
> Exception in thread "main" *java.lang.IllegalArgumentException*:
> *java.net.UnknownHostException*: infaqaha
>
>      at org.apache.hadoop.security.SecurityUtil.buildTokenService(
> *SecurityUtil.java:374*)
>
>      at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(
> *NameNodeProxies.java:312*)
>
>      at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(
> *NameNodeProxies.java:178*)
>
>      at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:662*)
>
>      at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:606*)
>
>      at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(
> *DistributedFileSystem.java:148*)
>
>      at org.apache.hadoop.fs.FileSystem.createFileSystem(
> *FileSystem.java:2596*)
>
>      at org.apache.hadoop.fs.FileSystem.access$200(*FileSystem.java:91*)
>
>      at org.apache.hadoop.fs.FileSystem$Cache.getInternal(
> *FileSystem.java:2630*)
>
>      at org.apache.hadoop.fs.FileSystem$Cache.get(*FileSystem.java:2612*)
>
>      at org.apache.hadoop.fs.FileSystem.get(*FileSystem.java:370*)
>
>      at org.apache.hadoop.fs.Path.getFileSystem(*Path.java:296*)
>
>      at org.apache.avro.mapred.FsInput.<init>(*FsInput.java:38*)
>
>      at parquetAndAvro.CopyOfInfaAvro.main(*CopyOfInfaAvro.java:97*)
>
> Caused by: *java.net.UnknownHostException*: infaqaha
>
>      ... 14 more
>
> ERROR: JDWP Unable to get JNI 1.2 environment, jvm->GetEnv() return code =
> -2
>
> JDWP exit error AGENT_ERROR_NO_JNI_ENV(183):  [util.c:840]
>
>
>
> *hdfs -site.xml:*
> <?xml version="1.0" encoding="UTF-8"?>
>
>
>
> <!--Autogenerated by Cloudera Manager-->
>
> <configuration>
>
>   <property>
>
>     <name>dfs.nameservices</name>
>
>     <value>INFAQAHA</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.client.failover.proxy.provider.INFAQAHA</name>
>
>
> <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.ha.automatic-failover.enabled.INFAQAHA</name>
>
>     <value>true</value>
>
>   </property>
>
>   <property>
>
>     <name>ha.zookeeper.quorum</name>
>
>     <value>ivlhdp18.informatica.com:2181</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.ha.namenodes.INFAQAHA</name>
>
>     <value>namenode34,namenode54</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.namenode.rpc-address.INFAQAHA.namenode34</name>
>
>     <value>ivlhdp18.informatica.com:8020</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode34</name>
>
>     <value>ivlhdp18.informatica.com:8022</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.namenode.http-address.INFAQAHA.namenode34</name>
>
>     <value>ivlhdp18.informatica.com:50070</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.namenode.https-address.INFAQAHA.namenode34</name>
>
>     <value>ivlhdp18.informatica.com:50470</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.namenode.rpc-address.INFAQAHA.namenode54</name>
>
>     <value>ivlhdp19.informatica.com:8020</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode54</name>
>
>     <value>ivlhdp19.informatica.com:8022</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.namenode.http-address.INFAQAHA.namenode54</name>
>
>     <value>ivlhdp19.informatica.com:50070</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.namenode.https-address.INFAQAHA.namenode54</name>
>
>     <value>ivlhdp19.informatica.com:50470</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.replication</name>
>
>     <value>1</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.blocksize</name>
>
>     <value>134217728</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.client.use.datanode.hostname</name>
>
>     <value>false</value>
>
>   </property>
>
>   <property>
>
>     <name>fs.permissions.umask-mode</name>
>
>     <value>022</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.namenode.acls.enabled</name>
>
>     <value>false</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.client.read.shortcircuit</name>
>
>     <value>false</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.domain.socket.path</name>
>
>     <value>/var/run/hdfs-sockets/dn</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.client.read.shortcircuit.skip.checksum</name>
>
>     <value>false</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.client.domain.socket.data.traffic</name>
>
>     <value>false</value>
>
>   </property>
>
>   <property>
>
>     <name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
>
>     <value>true</value>
>
>   </property>
>
> </configuration>
>
>
>
> *core-site.xml: *
>
> <?xml version="1.0" encoding="UTF-8"?>
>
>
>
> <!--Autogenerated by Cloudera Manager-->
>
> <configuration>
>
>   <property>
>
>     <name>fs.defaultFS</name>
>
>     <value>hdfs://INFAQAHA</value>
>
>   </property>
>
>   <property>
>
>     <name>fs.trash.interval</name>
>
>     <value>1</value>
>
>   </property>
>
>   <property>
>
>     <name>io.compression.codecs</name>
>
>
> <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.security.authentication</name>
>
>     <value>simple</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.security.authorization</name>
>
>     <value>false</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.rpc.protection</name>
>
>     <value>authentication</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.ssl.require.client.cert</name>
>
>     <value>false</value>
>
>     <final>true</final>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.ssl.keystores.factory.class</name>
>
>     <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
>
>     <final>true</final>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.ssl.server.conf</name>
>
>     <value>ssl-server.xml</value>
>
>     <final>true</final>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.ssl.client.conf</name>
>
>     <value>ssl-client.xml</value>
>
>     <final>true</final>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.security.auth_to_local</name>
>
>     <value>DEFAULT</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.oozie.hosts</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.oozie.groups</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.mapred.hosts</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.mapred.groups</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.flume.hosts</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.flume.groups</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.HTTP.hosts</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.HTTP.groups</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.hive.hosts</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.hive.groups</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.hue.hosts</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.hue.groups</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.httpfs.hosts</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.httpfs.groups</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.hdfs.groups</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.hdfs.hosts</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.yarn.hosts</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.proxyuser.yarn.groups</name>
>
>     <value>*</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.security.group.mapping</name>
>
>     <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
>
>   </property>
>
>   <property>
>
>     <name>hadoop.security.instrumentation.requires.admin</name>
>
>     <value>false</value>
>
>   </property>
>
> </configuration>
>
>
> Thanks,
> Mayank Mishra
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: user-unsubscribe@hadoop.apache.org
> For additional commands, e-mail: user-help@hadoop.apache.org
>



-- 
Best regards / Met vriendelijke groeten,

Niels Basjes

Mime
View raw message