hadoop-mapreduce-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Mayank Mishra <mayank.mishra.n...@gmail.com>
Subject Issue on accessing AVRO format file from HA Cluster.
Date Wed, 27 Apr 2016 14:27:54 GMT
Hi,

Kindly advise me to resolve the below exception getting while trying to
access the AVRO file on HA cluster from the below code.  I have attached
the resource file and pasted too.


*Code: **import* java.io.IOException;

*import* org.apache.avro.file.DataFileReader;

*import* org.apache.avro.file.FileReader;

*import* org.apache.avro.file.SeekableInput;

*import* org.apache.avro.generic.GenericDatumReader;

*import* org.apache.avro.generic.GenericRecord;

*import* org.apache.avro.io.DatumReader;

*import* org.apache.avro.mapred.FsInput;

*import* org.apache.hadoop.conf.Configuration;

*import* org.apache.hadoop.fs.Path;

*public* *static* *void* main(String args[]) *throws* IOException{

           Path path = *new* Path(
"hdfs://infaqaha/user/hive/warehouse/avro_table_string/000000_0");

           Configuration config = *new* Configuration(); // make this your
*Hadoop* *env* *config*

           config.addResource(
"C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\hadoop-conf\\hdfs-site.xml"
);

           config.addResource(
"C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\core-site");



           //*Metadata* fetch for each record, we sample only one record in
actual implementation.

           SeekableInput input = *new* FsInput(path, config); <-Exception
on this line

           DatumReader<GenericRecord> reader = *new*
GenericDatumReader<GenericRecord>();

           FileReader<GenericRecord> fileReader = DataFileReader.
*openReader*(input, reader);

fileReader.close(); // also closes underlying FsInput
}




*Exception is : *

Exception in thread "main" *java.lang.IllegalArgumentException*:
*java.net.UnknownHostException*: infaqaha

     at org.apache.hadoop.security.SecurityUtil.buildTokenService(
*SecurityUtil.java:374*)

     at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(
*NameNodeProxies.java:312*)

     at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(
*NameNodeProxies.java:178*)

     at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:662*)

     at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:606*)

     at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(
*DistributedFileSystem.java:148*)

     at org.apache.hadoop.fs.FileSystem.createFileSystem(
*FileSystem.java:2596*)

     at org.apache.hadoop.fs.FileSystem.access$200(*FileSystem.java:91*)

     at org.apache.hadoop.fs.FileSystem$Cache.getInternal(
*FileSystem.java:2630*)

     at org.apache.hadoop.fs.FileSystem$Cache.get(*FileSystem.java:2612*)

     at org.apache.hadoop.fs.FileSystem.get(*FileSystem.java:370*)

     at org.apache.hadoop.fs.Path.getFileSystem(*Path.java:296*)

     at org.apache.avro.mapred.FsInput.<init>(*FsInput.java:38*)

     at parquetAndAvro.CopyOfInfaAvro.main(*CopyOfInfaAvro.java:97*)

Caused by: *java.net.UnknownHostException*: infaqaha

     ... 14 more

ERROR: JDWP Unable to get JNI 1.2 environment, jvm->GetEnv() return code =
-2

JDWP exit error AGENT_ERROR_NO_JNI_ENV(183):  [util.c:840]



*hdfs -site.xml:*
<?xml version="1.0" encoding="UTF-8"?>



<!--Autogenerated by Cloudera Manager-->

<configuration>

  <property>

    <name>dfs.nameservices</name>

    <value>INFAQAHA</value>

  </property>

  <property>

    <name>dfs.client.failover.proxy.provider.INFAQAHA</name>


<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

  </property>

  <property>

    <name>dfs.ha.automatic-failover.enabled.INFAQAHA</name>

    <value>true</value>

  </property>

  <property>

    <name>ha.zookeeper.quorum</name>

    <value>ivlhdp18.informatica.com:2181</value>

  </property>

  <property>

    <name>dfs.ha.namenodes.INFAQAHA</name>

    <value>namenode34,namenode54</value>

  </property>

  <property>

    <name>dfs.namenode.rpc-address.INFAQAHA.namenode34</name>

    <value>ivlhdp18.informatica.com:8020</value>

  </property>

  <property>

    <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode34</name>

    <value>ivlhdp18.informatica.com:8022</value>

  </property>

  <property>

    <name>dfs.namenode.http-address.INFAQAHA.namenode34</name>

    <value>ivlhdp18.informatica.com:50070</value>

  </property>

  <property>

    <name>dfs.namenode.https-address.INFAQAHA.namenode34</name>

    <value>ivlhdp18.informatica.com:50470</value>

  </property>

  <property>

    <name>dfs.namenode.rpc-address.INFAQAHA.namenode54</name>

    <value>ivlhdp19.informatica.com:8020</value>

  </property>

  <property>

    <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode54</name>

    <value>ivlhdp19.informatica.com:8022</value>

  </property>

  <property>

    <name>dfs.namenode.http-address.INFAQAHA.namenode54</name>

    <value>ivlhdp19.informatica.com:50070</value>

  </property>

  <property>

    <name>dfs.namenode.https-address.INFAQAHA.namenode54</name>

    <value>ivlhdp19.informatica.com:50470</value>

  </property>

  <property>

    <name>dfs.replication</name>

    <value>1</value>

  </property>

  <property>

    <name>dfs.blocksize</name>

    <value>134217728</value>

  </property>

  <property>

    <name>dfs.client.use.datanode.hostname</name>

    <value>false</value>

  </property>

  <property>

    <name>fs.permissions.umask-mode</name>

    <value>022</value>

  </property>

  <property>

    <name>dfs.namenode.acls.enabled</name>

    <value>false</value>

  </property>

  <property>

    <name>dfs.client.read.shortcircuit</name>

    <value>false</value>

  </property>

  <property>

    <name>dfs.domain.socket.path</name>

    <value>/var/run/hdfs-sockets/dn</value>

  </property>

  <property>

    <name>dfs.client.read.shortcircuit.skip.checksum</name>

    <value>false</value>

  </property>

  <property>

    <name>dfs.client.domain.socket.data.traffic</name>

    <value>false</value>

  </property>

  <property>

    <name>dfs.datanode.hdfs-blocks-metadata.enabled</name>

    <value>true</value>

  </property>

</configuration>



*core-site.xml: *

<?xml version="1.0" encoding="UTF-8"?>



<!--Autogenerated by Cloudera Manager-->

<configuration>

  <property>

    <name>fs.defaultFS</name>

    <value>hdfs://INFAQAHA</value>

  </property>

  <property>

    <name>fs.trash.interval</name>

    <value>1</value>

  </property>

  <property>

    <name>io.compression.codecs</name>


<value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>

  </property>

  <property>

    <name>hadoop.security.authentication</name>

    <value>simple</value>

  </property>

  <property>

    <name>hadoop.security.authorization</name>

    <value>false</value>

  </property>

  <property>

    <name>hadoop.rpc.protection</name>

    <value>authentication</value>

  </property>

  <property>

    <name>hadoop.ssl.require.client.cert</name>

    <value>false</value>

    <final>true</final>

  </property>

  <property>

    <name>hadoop.ssl.keystores.factory.class</name>

    <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>

    <final>true</final>

  </property>

  <property>

    <name>hadoop.ssl.server.conf</name>

    <value>ssl-server.xml</value>

    <final>true</final>

  </property>

  <property>

    <name>hadoop.ssl.client.conf</name>

    <value>ssl-client.xml</value>

    <final>true</final>

  </property>

  <property>

    <name>hadoop.security.auth_to_local</name>

    <value>DEFAULT</value>

  </property>

  <property>

    <name>hadoop.proxyuser.oozie.hosts</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.oozie.groups</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.mapred.hosts</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.mapred.groups</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.flume.hosts</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.flume.groups</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.HTTP.hosts</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.HTTP.groups</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.hive.hosts</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.hive.groups</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.hue.hosts</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.hue.groups</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.httpfs.hosts</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.httpfs.groups</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.hdfs.groups</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.hdfs.hosts</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.yarn.hosts</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.proxyuser.yarn.groups</name>

    <value>*</value>

  </property>

  <property>

    <name>hadoop.security.group.mapping</name>

    <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>

  </property>

  <property>

    <name>hadoop.security.instrumentation.requires.admin</name>

    <value>false</value>

  </property>

</configuration>


Thanks,
Mayank Mishra

Mime
View raw message