Mailing-List: contact hive-user-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: hive-user@hadoop.apache.org
Received-SPF: pass (athena.apache.org: domain of edlinuxguru@gmail.com
 designates 209.85.160.48 as permitted sender)
DomainKey-Signature: a=rsa-sha1; c=nofws;
        d=gmail.com; s=gamma;
        h=mime-version:in-reply-to:references:date:message-id:subject:from:to
         :content-type:content-transfer-encoding;
        b=Bg+w8fZt2iwdgaI5iCtDf/cQlXY6sLaIWzpFUJeXu1Zu9lSGwKEelDpvoirMZY7geT
         gwHS0T9UZEUqAcw0AY/VIXKRdcPHjxWEbBu6Nwdl/0gL2f0fLBNFTvX8mHlETZDEd6Y5
         /pPJ+3WLmgHTmBMZE1TLn9az7YXixSRXny9no=
MIME-Version: 1.0
In-Reply-To: <i2qcbbf4b571004131442g3c22e772s69c8001557da868d@mail.gmail.com>
References: <o2lcbbf4b571004021416o5fbdb46bnd44d498528c08dde@mail.gmail.com>
	 <i2r34fd060d1004021834p89c5552ar79cfc7c2243c24c2@mail.gmail.com>
	 <i2qcbbf4b571004131442g3c22e772s69c8001557da868d@mail.gmail.com>
Date: Tue, 13 Apr 2010 20:15:41 -0400
Message-ID: <g2hcbbf4b571004131715ic04cd3ewa94aa2656cef584d@mail.gmail.com>
Subject: Re: Sequence Files with data inside key
From: Edward Capriolo <edlinuxguru@gmail.com>
To: "hive-user@hadoop.apache.org" <hive-user@hadoop.apache.org>
Content-Type: text/plain; charset=ISO-8859-1
Content-Transfer-Encoding: quoted-printable

I was looking at the code and it looks like hive uses
ignorekeyOUTPUTformat so rather the trying to swap values in the
inputformat just write an ignore value output format.

On Tuesday, April 13, 2010, Edward Capriolo <edlinuxguru@gmail.com> wrote:
>
>
> On Fri, Apr 2, 2010 at 9:34 PM, Zheng Shao <zshao9@gmail.com> wrote:
>
> The easiest way is to write a SequenceFileInputFormat that returns a
> RecordReader that has key in the value and value in the key.
>
> Zheng
>
> On Fri, Apr 2, 2010 at 2:16 PM, Edward Capriolo <edlinuxguru@gmail.com> w=
rote:
>> I have some sequence files in which all our data is in the key.
>>
>> http://osdir.com/ml/hive-user-hadoop-apache/2009-10/msg00027.html
>>
>> Has anyone tackled the above issue?
>>
>>
>
>
>
> --
> Yours,
> Zheng
>
>
> I am attempting to do this for sequence files. Unfortunately I have to co=
py much of the SequenceFile format since the reader (in) has private access=
.
> ----------------------------------------
> public class SequenceKeyOnlyInputFormat<K extends WritableComparable, V e=
xtends Writable> extends SequenceFileInputFormat<K, V> {
>
> =A0=A0=A0 public RecordReader<K, V> getRecordReader(InputSplit split, Job=
Conf job, Reporter reporter) throws IOException {
> =A0=A0=A0 =A0=A0=A0 reporter.setStatus(split.toString());
> =A0=A0=A0 =A0=A0=A0 return new SequenceKeyOnlyRecordReader<K, V>(job, (Fi=
leSplit) split);
> =A0=A0=A0 }
>
> }
> --------------------------------------------
> @SuppressWarnings({ "unchecked", "deprecation" })
> public class SequenceKeyOnlyRecordReader<K extends WritableComparable , V=
 extends Writable>
> implements RecordReader<K, V>{
>
> =A0=A0=A0 private SequenceFile.Reader in;
> =A0=A0=A0 private long start;
> =A0=A0=A0 private long end;
> =A0=A0=A0 private boolean more =3D true;
> =A0=A0=A0 protected Configuration conf;
>
>
> =A0=A0=A0 public SequenceKeyOnlyRecordReader(Configuration conf, FileSpli=
t split) throws IOException {
> =A0=A0=A0 =A0=A0=A0 Path path =3D split.getPath();
> =A0=A0=A0 =A0=A0=A0 FileSystem fs =3D path.getFileSystem(conf);
> =A0=A0=A0 =A0=A0=A0 this.in =3D new SequenceFile.Reader(fs, path, conf);
> =A0=A0=A0 =A0=A0=A0 this.end =3D split.getStart() + split.getLength();
> =A0=A0=A0 =A0=A0=A0 this.conf =3D conf;
>
> =A0=A0=A0 =A0=A0=A0 if (split.getStart() > in.getPosition()) in.sync(spli=
t.getStart()); // sync to start
>
> =A0=A0=A0 =A0=A0=A0 this.start =3D in.getPosition();
> =A0=A0=A0 =A0=A0=A0 more =3D start < end;
> =A0=A0=A0 }
>
> =A0=A0=A0 /**
> =A0=A0=A0 =A0* The class of key that must be passed to {@link #next(Objec=
t, Object)}..
> =A0=A0=A0 =A0*/
> =A0=A0=A0 public Class getKeyClass() {
> =A0=A0=A0 =A0=A0=A0 return in.getKeyClass();
> =A0=A0=A0 }
>
> =A0=A0=A0 /**
> =A0=A0=A0 =A0* The class of value that must be passed to {@link #next(Obj=
ect, Object)}..
> =A0=A0=A0 =A0*/
> =A0=A0=A0 public Class getValueClass() {
> =A0=A0=A0 =A0=A0=A0 return in.getKeyClass();
> =A0=A0=A0 }
>
> =A0=A0=A0 public K createKey() {
> =A0=A0=A0 =A0=A0=A0 return (K) ReflectionUtils.newInstance(getKeyClass(),=
 conf);
> =A0=A0=A0 }
>
> =A0=A0=A0 public V createValue() {
> =A0=A0=A0 =A0=A0=A0 return (V) ReflectionUtils.newInstance(getKeyClass(),=
 conf);
> =A0=A0=A0 }
>
> =A0=A0=A0 public synchronized boolean next(K key, V value) throws IOExcep=
tion {
> =A0=A0=A0 =A0=A0=A0 if (!more) return false;
> =A0=A0=A0 =A0=A0=A0 long pos =3D in.getPosition();
>
> =A0=A0=A0 =A0=A0=A0 boolean remaining =3D in.next(key);
> =A0=A0=A0 =A0=A0=A0 if (remaining) {
> =A0=A0=A0 =A0=A0=A0 =A0=A0=A0 getCurrentValue(value);
> =A0=A0=A0 =A0=A0=A0 }
> =A0=A0=A0 =A0=A0=A0 if (pos >=3D end && in.syncSeen()) {
> =A0=A0=A0 =A0=A0=A0 =A0=A0=A0 more =3D false;
> =A0=A0=A0 =A0=A0=A0 } else {
> =A0=A0=A0 =A0=A0=A0 =A0=A0=A0 more =3D remaining;
> =A0=A0=A0 =A0=A0=A0 }
> =A0=A0=A0 =A0=A0=A0 return more;
> =A0=A0=A0 }
>
> =A0=A0=A0 protected synchronized boolean next(K key) throws IOException {
> =A0=A0=A0 =A0=A0=A0 if (!more) return false;
> =A0=A0=A0 =A0=A0=A0 long pos =3D in.getPosition();
> =A0=A0=A0 =A0=A0=A0 boolean remaining =3D in.next(key);
> =A0=A0=A0 =A0=A0=A0 if (pos >=3D end && in.syncSeen()) {
> =A0=A0=A0 =A0=A0=A0 =A0=A0=A0 more =3D false;
> =A0=A0=A0 =A0=A0=A0 } else {
> =A0=A0=A0 =A0=A0=A0 =A0=A0=A0 more =3D remaining;
> =A0=A0=A0 =A0=A0=A0 }
> =A0=A0=A0 =A0=A0=A0 return more;
> =A0=A0=A0 }
>
> =A0=A0=A0 protected synchronized void getCurrentValue(V value) throws IOE=
xception {
> =A0=A0=A0 =A0=A0=A0 =A0in.getCurrentValue(value);
> =A0=A0=A0 =A0=A0=A0 //in.next(value);
> =A0=A0=A0 }
>
> =A0=A0=A0 /**
> =A0=A0=A0 =A0* Return the progress within the input split
> =A0=A0=A0 =A0*
> =A0=A0=A0 =A0* @return 0.0 to 1.0 of the input byte range
> =A0=A0=A0 =A0*/
> =A0=A0=A0 public float getProgress() throws IOException {
> =A0=A0=A0 =A0=A0=A0 if (end =3D=3D start) {
> =A0=A0=A0 =A0=A0=A0 =A0=A0=A0 return 0.0f;
> =A0=A0=A0 =A0=A0=A0 } else {
> =A0=A0=A0 =A0=A0=A0 =A0=A0=A0 return Math.min(1.0f, (in.getPosition() - s=
tart) / (float) (end - start));
> =A0=A0=A0 =A0=A0=A0 }
> =A0=A0=A0 }
>
> =A0=A0=A0 public synchronized long getPos() throws IOException {
> =A0=A0=A0 =A0=A0=A0 return in.getPosition();
> =A0=A0=A0 }
>
> =A0=A0=A0 protected synchronized void seek(long pos) throws IOException {
> =A0=A0=A0 =A0=A0=A0 in.seek(pos);
> =A0=A0=A0 }
>
> =A0=A0=A0 public synchronized void close() throws IOException {
> =A0=A0=A0 =A0=A0=A0 in.close();
> =A0=A0=A0 }
>
> }
>
> seems like:
>
> =A0=A0=A0 protected synchronized void getCurrentValue(V value) throws IOE=
xception {
> =A0=A0=A0 =A0=A0=A0 =A0in.getCurrentValue(value);
> =A0=A0=A0 }
>
> ^ Returns nulls
>
> =A0=A0=A0 protected synchronized void getCurrentValue(V value) throws IOE=
xception {
> =A0=A0=A0 =A0=A0 in.next(value);
> =A0=A0=A0 }
>
> ^ returns every other row.
>
> Do you have any idea what I am doing wrong? Will contrib it hopefully If =
i can get this going correctly.
>
> Thanks,
> Edward
>