Mailing-List: contact user-help@hive.apache.org; run by ezmlm
Precedence: bulk
Reply-To: user@hive.apache.org
Received-SPF: pass (athena.apache.org: domain of smallputao@gmail.com
 designates 209.85.220.176 as permitted sender)
MIME-Version: 1.0
Date: Tue, 17 Jan 2012 19:39:51 +0800
Message-ID: 
 <CAE3XUg4A8-e+143avemmMRurHF2KPwWJp82hVc3xf7S2RmvXhQ@mail.gmail.com>
Subject: Cannot create an instance of InputFormat
From: Bing Li <smallputao@gmail.com>
To: user@hive.apache.org
Content-Type: multipart/alternative; boundary=f46d043892910cb4a604b6b7caf0

--f46d043892910cb4a604b6b7caf0
Content-Type: text/plain; charset=ISO-8859-1

My Steps:
I define a class "public class myInputFormat extends TextInputFormat
implements JobConfigurable" to specify input format.

hive> add jar /home/biadmin/hiveudf/myFileFormat.jar;
Added /home/biadmin/hiveudf/myFileFormat.jar to class path
Added resource: /home/biadmin/hiveudf/myFileFormat.jar

hive> list jars;
/home/biadmin/hiveudf/myFileFormat.jar

hive> create table IOtable(str1 string, str2 string, str3 string) stored as
INPUTFORMAT 'com.mytest.fileformat.myInputFormat' OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat' ;
OK
Time taken: 0.081 seconds

hive> load data local inpath '/home/biadmin/hivetbl/IOtable_data.txt' into
table IOtable;
Copying data from file:/home/biadmin/hivetbl/IOtable_data.txt
Copying file: file:/home/biadmin/hivetbl/IOtable_data.txt
Loading data to table default.iotable
OK
Time taken: 0.147 seconds

hive>  select * from IOtable;
OK
Failed with exception java.io.IOException:java.io.IOException: Cannot
create an instance of InputFormat class com.mytest.fileformat.myInputFormat
as specified in mapredWork!
Time taken: 0.059 seconds


*Here is my source code :*
===============================
package com.mytest.fileformat;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobConfigurable;
import org.apache.hadoop.mapred.LineRecordReader;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.TextInputFormat;

@SuppressWarnings("deprecation")
public class myInputFormat extends TextInputFormat implements
JobConfigurable {
 TextInputFormat format;
    JobConf job;

public myInputFormat() {
        format = new TextInputFormat();
    }

 @Override
public void configure(JobConf job) {
        this.job = job;
        format.configure(job);
}
    public RecordReader<LongWritable, Text> getRecordReader(
            InputSplit genericSplit, JobConf job, Reporter reporter)
            throws IOException {

        reporter.setStatus(genericSplit.toString());
        return new myLineRecordReader(job, (FileSplit) genericSplit);
    }


    public static class myLineRecordReader implements
    RecordReader<LongWritable, Text> {
      LineRecordReader lineReader;
      LongWritable lineKey;
      Text lineValue;

      public myLineRecordReader(JobConf job, FileSplit split) throws
IOException {
        lineReader = new LineRecordReader(job, split);
        lineKey = lineReader.createKey();
        lineValue = lineReader.createValue();
      }

      public boolean next(LongWritable key, Text value) throws IOException {
        while (lineReader.next(lineKey, lineValue)) {
          String strReplace = lineValue.toString().toLowerCase().replace(
"$$$$" , "\001" );
          Text txtReplace = new Text();
          txtReplace.set(strReplace);
          value.set(txtReplace.getBytes(), 0, txtReplace.getLength());
          return true ;
         }
         // no more data
         return false;
      }  /** end next **/


      public LongWritable createKey() {
        return lineReader.createKey();
      }
      public Text createValue() {
        return lineReader.createValue();
      }
      public long getPos() throws IOException{
        return lineReader.getPos();
      }
      public float getProgress() throws IOException{
        return lineReader.getProgress();
      }
      public void close() throws IOException{
        lineReader.close();
      }
     }  /** end class myLineRecordReader **/
}

--f46d043892910cb4a604b6b7caf0
Content-Type: text/html; charset=ISO-8859-1
Content-Transfer-Encoding: quoted-printable

<div>My Steps:</div><div>I define a class &quot;public class myInputFormat =
extends TextInputFormat implements JobConfigurable&quot; to specify input f=
ormat.=A0</div><div><br></div><div>hive&gt; add jar /home/biadmin/hiveudf/m=
yFileFormat.jar;</div>
<div>Added /home/biadmin/hiveudf/myFileFormat.jar to class path</div><div>A=
dded resource: /home/biadmin/hiveudf/myFileFormat.jar</div><div><br></div><=
div>hive&gt; list jars;</div><div>/home/biadmin/hiveudf/myFileFormat.jar</d=
iv>
<div><br></div><div>hive&gt; create table IOtable(str1 string, str2 string,=
 str3 string) stored as INPUTFORMAT &#39;com.mytest.fileformat.myInputForma=
t&#39; OUTPUTFORMAT &#39;org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFo=
rmat&#39; ;</div>
<div>OK</div><div>Time taken: 0.081 seconds</div><div><br></div><div>hive&g=
t; load data local inpath &#39;/home/biadmin/hivetbl/IOtable_data.txt&#39; =
into table IOtable;</div><div>Copying data from file:/home/biadmin/hivetbl/=
IOtable_data.txt</div>
<div>Copying file: file:/home/biadmin/hivetbl/IOtable_data.txt</div><div>Lo=
ading data to table default.iotable</div><div>OK</div><div>Time taken: 0.14=
7 seconds</div><div><br></div><div>hive&gt; =A0select * from IOtable;</div>
<div>OK</div><div>Failed with exception java.io.IOException:java.io.IOExcep=
tion: Cannot create an instance of InputFormat class com.mytest.fileformat.=
myInputFormat as specified in mapredWork!</div><div>Time taken: 0.059 secon=
ds</div>
<div><br></div><div><br></div><div><br></div><div><br></div><div><b>Here is=
 my source code :</b></div><div>=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D</div><div><div>package =
com.mytest.fileformat;</div><div><br></div><div>import java.io.IOException;=
 =A0</div>
<div>=A0=A0</div><div>import org.apache.hadoop.io.LongWritable; =A0</div><d=
iv>import org.apache.hadoop.io.Text; =A0</div><div>import org.apache.common=
s.logging.Log;</div><div>import org.apache.commons.logging.LogFactory;</div=
><div>
import org.apache.hadoop.mapred.FileSplit; =A0</div><div>import org.apache.=
hadoop.mapred.InputSplit; =A0</div><div>import org.apache.hadoop.mapred.Job=
Conf; =A0</div><div>import org.apache.hadoop.mapred.JobConfigurable;=A0</di=
v><div>
import org.apache.hadoop.mapred.LineRecordReader;</div><div>import org.apac=
he.hadoop.mapred.RecordReader; =A0</div><div>import org.apache.hadoop.mapre=
d.Reporter; =A0</div><div>import org.apache.hadoop.mapred.InputFormat;</div=
>
<div>import org.apache.hadoop.mapred.TextInputFormat; =A0</div><div><br></d=
iv><div>@SuppressWarnings(&quot;deprecation&quot;)</div><div>public class m=
yInputFormat extends TextInputFormat implements JobConfigurable {</div><div=
>
<span class=3D"Apple-tab-span" style=3D"white-space:pre">	</span></div><div=
><span class=3D"Apple-tab-span" style=3D"white-space:pre">	</span>TextInput=
Format format;</div><div>=A0 =A0 JobConf job;</div><div>=A0 =A0=A0</div><di=
v><span class=3D"Apple-tab-span" style=3D"white-space:pre">	</span>public m=
yInputFormat() {</div>
<div>=A0 =A0 =A0 =A0 format =3D new TextInputFormat();</div><div>=A0 =A0 }<=
/div><div>=A0</div><div><span class=3D"Apple-tab-span" style=3D"white-space=
:pre">	</span> @Override</div><div><span class=3D"Apple-tab-span" style=3D"=
white-space:pre">	</span>public void configure(JobConf job) {</div>
<div><span class=3D"Apple-tab-span" style=3D"white-space:pre">	</span> =A0 =
=A0 =A0 =A0this.job =3D job;</div><div><span class=3D"Apple-tab-span" style=
=3D"white-space:pre">	</span> =A0 =A0 =A0 =A0format.configure(job);</div><d=
iv><span class=3D"Apple-tab-span" style=3D"white-space:pre">	</span>}</div>
<div>=A0 =A0 public RecordReader&lt;LongWritable, Text&gt; getRecordReader(=
 =A0</div><div>=A0 =A0 =A0 =A0 =A0 =A0 InputSplit genericSplit, JobConf job=
, Reporter reporter) =A0</div><div>=A0 =A0 =A0 =A0 =A0 =A0 throws IOExcepti=
on { =A0</div><div>=A0=A0</div><div>
=A0 =A0 =A0 =A0 reporter.setStatus(genericSplit.toString()); =A0</div><div>=
=A0 =A0 =A0 =A0 return new myLineRecordReader(job, (FileSplit) genericSplit=
); =A0</div><div>=A0 =A0 } =A0</div><div><br></div><div><span class=3D"Appl=
e-tab-span" style=3D"white-space:pre">	</span> =A0</div>
<div>=A0 =A0 public static class myLineRecordReader implements</div><div>=
=A0 =A0 RecordReader&lt;LongWritable, Text&gt; {</div><div><span class=3D"A=
pple-tab-span" style=3D"white-space:pre">	</span></div><div>=A0 =A0 =A0 Lin=
eRecordReader lineReader;</div>
<div>=A0 =A0 =A0 LongWritable lineKey;</div><div>=A0 =A0 =A0 Text lineValue=
;</div><div><br></div><div>=A0 =A0 =A0 public myLineRecordReader(JobConf jo=
b, FileSplit split) throws IOException {</div><div>=A0 =A0 =A0 =A0 lineRead=
er =3D new LineRecordReader(job, split);</div>
<div>=A0 =A0 =A0 =A0 lineKey =3D lineReader.createKey();</div><div>=A0 =A0 =
=A0 =A0 lineValue =3D lineReader.createValue();</div><div>=A0 =A0 =A0 }</di=
v><div>=A0 =A0=A0</div><div>=A0 =A0 =A0 public boolean next(LongWritable ke=
y, Text value) throws IOException {</div>
<div>=A0 =A0 =A0 =A0 while (lineReader.next(lineKey, lineValue)) {</div><di=
v>=A0 =A0 =A0 =A0 =A0 String strReplace =3D lineValue.toString().toLowerCas=
e().replace( &quot;$$$$&quot; , &quot;\001&quot; );</div><div>=A0 =A0 =A0 =
=A0 =A0 Text txtReplace =3D new Text();</div>
<div>=A0 =A0 =A0 =A0 =A0 txtReplace.set(strReplace);</div><div>=A0 =A0 =A0 =
=A0 =A0 value.set(txtReplace.getBytes(), 0, txtReplace.getLength());</div><=
div>=A0 =A0 =A0 =A0 =A0 return true ;</div><div>=A0 =A0 =A0 =A0 =A0}</div><=
div>=A0 =A0 =A0 =A0 =A0// no more data</div><div>
=A0 =A0 =A0 =A0 =A0return false; =A0 =A0=A0</div><div>=A0 =A0 =A0 } =A0/** =
end next **/</div><div>=A0 =A0 =A0=A0</div><div>=A0 =A0=A0</div><div>=A0 =
=A0 =A0 public LongWritable createKey() {</div><div>=A0 =A0 =A0 =A0 return =
lineReader.createKey();</div><div>=A0 =A0 =A0 }</div><div>
=A0 =A0 =A0 public Text createValue() {</div><div>=A0 =A0 =A0 =A0 return li=
neReader.createValue();</div><div>=A0 =A0 =A0 }</div><div>=A0 =A0 =A0 publi=
c long getPos() throws IOException{</div><div>=A0 =A0 =A0 =A0 return lineRe=
ader.getPos();</div><div>=A0 =A0 =A0 }</div>
<div>=A0 =A0 =A0 public float getProgress() throws IOException{</div><div>=
=A0 =A0 =A0 =A0 return lineReader.getProgress();</div><div>=A0 =A0 =A0 }</d=
iv><div>=A0 =A0 =A0 public void close() throws IOException{</div><div>=A0 =
=A0 =A0 =A0 lineReader.close();</div>
<div>=A0 =A0 =A0 }</div><div>=A0 =A0 =A0} =A0/** end class myLineRecordRead=
er **/</div><div>}</div></div>

--f46d043892910cb4a604b6b7caf0--