hadoop-common-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Aaron Baff <Aaron.B...@telescope.tv>
Subject Problem with custom WritableComparable
Date Fri, 12 Nov 2010 00:29:15 GMT
I'm having a problem with a custom WritableComparable that I created to use as a Key object.
I basically have a number of identifier's with a timestamp, and I'm wanting to group the Identifier's
together in the reducer, and order the records by the timestamp (oldest to newest). When I
used it as coded, I end up with a new reduce() call for every single record (even with the
same Identifier), while when I comment out the timestamp comparison code (as below), it works
perfectly and I only see 1 reduce() call per Identifier as I'm just returning the compareTo()
of the 2 Identifiers. Have I made a wrong assumption somewhere about how it's supposed to
work? Did I do something wrong?

--Aaron


public class IdentifierTimestampKey implements WritableComparable {
    private String identifier = "";
    private long timestamp = 0L;

    public void write(DataOutput out) throws IOException {
        out.writeUTF(identifier);
        out.writeLong(timestamp);
    }

    public void readFields(DataInput in) throws IOException {
        identifier = in.readUTF();
        timestamp = in.readLong();
    }

    @Override
    public int hashCode() {
        return identifier.hashCode();
    }

    @Override
    public boolean equals(Object obj) {
        if (obj == null) {
            return false;
        }
        if( obj instanceof IdentifierTimestampKey ) {
            final IdentifierTimestampKey other = (IdentifierTimestampKey) obj;
            if( this.identifier == null ) {
                return other.identifier == null;
            } else {
                return this.identifier.equals(other.identifier);
            }
        }
        return false;
    }

    public int compareTo(Object obj) {
        if (obj == null) {
            throw new ClassCastException("Object is NULL and so cannot be compared!");
        }
        int ret = 0;
        if( obj instanceof IdentifierTimestampKey ) {
            final IdentifierTimestampKey other = (IdentifierTimestampKey)obj;
//            if( this.identifier.equals(other.identifier) ) {
//                if( this.timestamp < other.timestamp ) {
//                    ret = -1;
//                } else if( this.timestamp > other.timestamp ) {
//                    ret = 1;
//                }
//            } else {
                ret = this.identifier.compareTo(other.identifier);
//            }
        } else {
            throw new ClassCastException("Object is of type " + obj.getClass().getName() +
" which cannot be compared to this class of type " + getClass().getName());
        }

        return ret;
    }

    public String getIdentifier() {
        return identifier;
    }

    public void setIdentifier(String identifier) {
        this.identifier = identifier;
    }

    public long getTimestamp() {
        return timestamp;
    }

    public void setTimestamp(long timestamp) {
        this.timestamp = timestamp;
    }

    public static class Comparator extends WritableComparator {
        public Comparator() {
          super(IdentifierTimestampKey.class, true);
        }
    }
}


Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message