hadoop-common-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Camilo Arango" <carango.develo...@gmail.com>
Subject Help with custom key Type
Date Sun, 02 Dec 2007 01:37:56 GMT
Hi,
I am trying to define a custom key type in hadoop (version 0.15.0)

This is how my class looks like:

public class ClassAttributeValueKey implements WritableComparable {

 public int classification;

public int attribute;

public int value;


 public ClassAttributeValueKey() {

 }

 public ClassAttributeValueKey(int classification, int attribute, int value)
{

this.attribute = attribute;

this.value = value;

this.classification = classification;

}


 /* (non-Javadoc)

 * @see org.apache.hadoop.io.Writable#readFields(java.io.DataInput)

 */

public void readFields(DataInput in) throws IOException {

attribute = in.readInt();

value = in.readInt();

classification = in.readInt();

}


 /* (non-Javadoc)

 * @see org.apache.hadoop.io.Writable#write(java.io.DataOutput)

 */

public void write(DataOutput out) throws IOException {

out.write(attribute);

out.write(value);

out.write(classification);

}


 /* (non-Javadoc)

 * @see java.lang.Comparable#compareTo(java.lang.Object)

 */

public int compareTo(Object obj) {

ClassAttributeValueKey o = (ClassAttributeValueKey) obj;

int dif = classification - o.classification;

if(dif == 0){

int dif2 = attribute - o.attribute;

if(dif2 == 0){

return value - o.value;

}

return dif2;

}

return dif;

}

 public int getClassification() {

return classification;

}


 public int getAttribute() {

return attribute;

}


 public int getValue() {

return value;

}


 public String toString(){

return String.format("{class: %d attribute: %d value: %d}", classification,
attribute, value);

}


 @Override

public int hashCode() {

final int prime = 31;

int result = 1;

result = prime * result + attribute;

result = prime * result + classification;

result = prime * result + value;

return result;

}


 @Override

public boolean equals(Object obj) {

if (this == obj)

return true;

if (obj == null)

return false;

if (getClass() != obj.getClass())

return false;

final ClassAttributeValueKey other = (ClassAttributeValueKey) obj;

if (attribute != other.attribute)

return false;

if (classification != other.classification)

return false;

if (value != other.value)

return false;

return true;

}


}


When I try to use it as the key on the output of my mapper, I get the
following error:


java.lang.RuntimeException: java.io.EOFException
at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java
:97)
at cmput681.ClassAttributeValueKey$Comparator.compare(
ClassAttributeValueKey.java:123)
at org.apache.hadoop.mapred.BasicTypeSorterBase.compare(
BasicTypeSorterBase.java:133)
at org.apache.hadoop.mapred.MergeSorter.compare(MergeSorter.java:59)
at org.apache.hadoop.mapred.MergeSorter.compare(MergeSorter.java:35)
at org.apache.hadoop.util.MergeSort.mergeSort(MergeSort.java:46)
at org.apache.hadoop.util.MergeSort.mergeSort(MergeSort.java:55)
at org.apache.hadoop.util.MergeSort.mergeSort(MergeSort.java:55)
at org.apache.hadoop.util.MergeSort.mergeSort(MergeSort.java:55)
at org.apache.hadoop.util.MergeSort.mergeSort(MergeSort.java:55)
at org.apache.hadoop.util.MergeSort.mergeSort(MergeSort.java:55)
at org.apache.hadoop.util.MergeSort.mergeSort(MergeSort.java:55)
at org.apache.hadoop.util.MergeSort.mergeSort(MergeSort.java:55)
at org.apache.hadoop.util.MergeSort.mergeSort(MergeSort.java:55)
at org.apache.hadoop.util.MergeSort.mergeSort(MergeSort.java:55)
at org.apache.hadoop.util.MergeSort.mergeSort(MergeSort.java:55)
at org.apache.hadoop.mapred.MergeSorter.sort(MergeSorter.java:46)
at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.sortAndSpillToDisk(
MapTask.java:396)
at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.flush(MapTask.java:604)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:193)
at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:132)
Caused by: java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:358)
at cmput681.ClassAttributeValueKey.readFields(ClassAttributeValueKey.java
:40)
at org.apache.hadoop.io.WritableComparator.compare(WritableComparator.java
:91)
... 20 more
Exception in thread "main" java.io.IOException: Job failed!
at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:831)
at cmput681.NaiveBayesTool.run(NaiveBayesTool.java:38)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65)
at cmput681.NaiveBayesMain.main(NaiveBayesMain.java:9)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java
:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(
DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.apache.hadoop.util.RunJar.main(RunJar.java:155)


I donĀ“t know if the problem is the implementation of my key class. Please
help me to fix it.

Thanks,

C. Arango.

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message