hadoop-common-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Owen O'Malley <o...@yahoo-inc.com>
Subject Re: Hadoop for perl
Date Fri, 23 Mar 2007 21:17:55 GMT
Sorry, it looks like the list ate my attachment

-- Owen

#ifndef HADOOP_PIPES_HH
#define HADOOP_PIPES_HH

#ifdef SWIG
%module (directors="1") HadoopPipes
%include "std_string.i"
%feature("director") Mapper;
%feature("director") Reducer;
%feature("director") Partitioner;
%feature("director") RecordReader;
%feature("director") RecordWriter;
%feature("director") Factory;
#else
#include <string>
#endif

namespace HadoopPipes {

/**
* This interface defines the interface between application code and the
* foreign code interface to Hadoop Map/Reduce.
*/

/**
* A JobConf defines the properties for a job.
*/
class JobConf {
public:
   virtual bool hasKey(const std::string& key) const = 0;
   virtual const std::string& get(const std::string& key) const = 0;
   virtual int getInt(const std::string& key) const = 0;
   virtual float getFloat(const std::string& key) const = 0;
   virtual bool getBoolean(const std::string&key) const = 0;
   virtual ~JobConf() {}
};

/**
* Task context provides the information about the task and job.
*/
class TaskContext {
public:
   /**
    * Get the JobConf for the current task.
    */
   virtual const JobConf* getJobConf() = 0;

   /**
    * Get the current key.
    * @return the current key
    */
   virtual const std::string& getInputKey() = 0;

   /**
    * Get the current value.
    * @return the current value
    */
   virtual const std::string& getInputValue() = 0;

   /**
    * Generate an output record
    */
   virtual void emit(const std::string& key, const std::string&  
value) = 0;

   /**
    * Mark your task as having made progress without changing the status
    * message.
    */
   virtual void progress() = 0;

   /**
    * Set the status message and call progress.
    */
   virtual void setStatus(const std::string& status) = 0;

   /**
    * Get the name of the key class of the input to this task.
    */
   virtual const std::string& getInputKeyClass() = 0;

   /**
    * Get the name of the value class of the input to this task.
    */
   virtual const std::string& getInputValueClass() = 0;

   virtual ~TaskContext() {}
};

class MapContext: public TaskContext {
public:

   /**
    * Access the InputSplit of the mapper.
    */
   virtual const std::string& getInputSplit() = 0;

};

class ReduceContext: public TaskContext {
public:
   /**
    * Advance to the next value.
    */
   virtual bool nextValue() = 0;
};

class Closable {
public:
   virtual void close() {}
   virtual ~Closable() {}
};

/**
* The application's mapper class to do map.
*/
class Mapper: public Closable {
public:
   virtual void map(MapContext& context) = 0;
};

/**
* The application's reducer class to do reduce.
*/
class Reducer: public Closable {
public:
   virtual void reduce(ReduceContext& context) = 0;
};

/**
* User code to decide where each key should be sent.
*/
class Partitioner {
public:
   virtual int partition(const std::string& key, int numOfReduces) = 0;
   virtual ~Partitioner() {}
};

/**
* For applications that want to read the input directly for the map  
function
* they can define RecordReaders in C++.
*/
class RecordReader: public Closable {
public:
   virtual bool next(std::string& key, std::string& value) = 0;

   /**
    * The progress of the record reader through the split as a value  
between
    * 0.0 and 1.0.
    */
   virtual float getProgress() = 0;
};

/**
* An object to write key/value pairs as they are emited from the reduce.
*/
class RecordWriter: public Closable {
public:
   virtual void emit(const std::string& key,
                     const std::string& value) = 0;
};

/**
* A factory to create the necessary application objects.
*/
class Factory {
public:
   virtual Mapper* createMapper(MapContext& context) const = 0;
   virtual Reducer* createReducer(ReduceContext& context) const = 0;

   /**
    * Create a combiner, if this application has one.
    * @return the new combiner or NULL, if one is not needed
    */
   virtual Reducer* createCombiner(MapContext& context) const {
     return NULL;
   }

   /**
    * Create an application partitioner object.
    * @return the new partitioner or NULL, if the default partitioner  
should be
    *     used.
    */
   virtual Partitioner* createPartitioner(MapContext& context) const {
     return NULL;
   }

   /**
    * Create an application record reader.
    * @return the new RecordReader or NULL, if the Java RecordReader  
should be
    *    used.
    */
   virtual RecordReader* createRecordReader(MapContext& context) const {
     return NULL;
   }

   /**
    * Create an application record writer.
    * @return the new RecordWriter or NULL, if the Java RecordWriter  
should be
    *    used.
    */
   virtual RecordWriter* createRecordWriter(ReduceContext& context)  
const {
     return NULL;
   }

   virtual ~Factory() {}
};

/**
* Start the event handling loop that runs the task. This will use the  
given
* factory to create Mappers and Reducers and so on.
* @return true, if the task succeeded.
*/
bool runTask(const Factory& factory);

}

#endif


Mime
View raw message