hadoop-common-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ykj <ykj...@163.com>
Subject Hama Problem
Date Wed, 20 May 2009 07:08:52 GMT

Hello,everyone 

       I am new to hama. in our project ,my team leader let me upload  old
code, run it on hadoop with parallel matrix computation.this is old code:

public class EigenFaceGenerator {
	
	Matrix 	averageFace;		//stores the average face useful when probing the
database
    Matrix 	eigVectors;			//stores all the sorted eigen vectors from the
training set
	Matrix 	eigValues;			//Stores all the sorted eigen Values from the training
set
	boolean trained = false;	//has a training set been provided yet?
	int 	numEigenVecs = 0;	//number of eigen vectors availiable
	
	/**
	 * 
	 * @param faces array of pictures to be used for the training
	 * @param progress 
	 */
	public void processTrainingSet(Face[] faces)
	{
		//TODO : there are errors that can be thrown when no fiels are parsed into
here
		//TODO : Check that all the images are the same size

		/**
		 * STEP 1
		 * 		Read in the images, flatten them out into one row of values, and
stack in a big matrix
		 */
		double[][] dpix = new
double[faces.length][faces[0].picture.getImagePixels().length];
		
		for(int i=0; i<faces.length; i++)
		{		//for each picture in the set
			double[] pixels = faces[i].picture.getImagePixels();
			for (int j=0; j<pixels.length; j++)
			{
				dpix[i][j] = pixels[j];
			}
		}
		//make matrix of stacked flattened images
		Matrix matrix = new Matrix(dpix);
		
		
		/**
		 * STEP 2
		 * 		Calculate the average face, and then take this away from each of the
image
		 * 		effectivly calculating the difference form the average.
		 */
		//compute the average image
		averageFace = new Matrix(1,matrix.getColumnDimension());
		for(int i=0; i<matrix.getRowDimension(); i++)
		{
			averageFace.plusEquals(matrix.getMatrix(i,i,0,matrix.getColumnDimension()
- 1));
		}
		averageFace.timesEquals(1.0/(double)matrix.getRowDimension());	//divide by
the number of pixels to get the average
		Matrix bigAvg = new Matrix(matrix.getRowDimension(),
matrix.getColumnDimension());
		for (int i=0; i<bigAvg.getRowDimension(); i++)
		{
			bigAvg.setMatrix(i,i,0,bigAvg.getColumnDimension()-1,averageFace);
		}
		// Compute the diference form the average face for each image
		Matrix A = matrix.minus(bigAvg).transpose();
		
		
		/**
		 * STEP 3
		 * 		Now compute the the patternwise (nexamp x nexamp) covariance matrix
		 */		
		// TODO : for the presentation work out why this is done, and what it's
telling us
		Matrix At = A.transpose();
	    Matrix L = At.times(A);

		
		/**
		 * STEP 4
		 * 		Calculate the eigen values and vectors of this covariance matrix
		 * 
		 * 		% Get the eigenvectors (columns of Vectors) and eigenvalues (diag of
Values)
		 */
		EigenvalueDecomposition eigen = L.eig();
		eigValues 	= eigen.getD();
		eigVectors 	= eigen.getV();
		
		
		/**
		 * STEP 5
		 * 		% Sort the vectors/values according to size of eigenvalue
		 */
		Matrix[] eigDVSorted = sortem(eigValues, eigVectors);
		eigValues = eigDVSorted[0];
		eigVectors = eigDVSorted[1];
			
		
		/**
		 * STEP 6
		 * 		% Convert the eigenvectors of A'*A into eigenvectors of A*A'
		 */

		eigVectors = A.times(eigVectors);
		
		
		/**
		 * STEP 7
		 * 		% Get the eigenvalues out of the diagonal matrix and
		 *		% normalize them so the evalues are specifically for cov(A'), not
A*A'.
		 */
		double[] values = diag(eigValues);
		for(int i = 0; i < values.length; i++)
			values[i] /= A.getColumnDimension() - 1;
		
		
		/**
		 * STEP 8
		 * 		% Normalize Vectors to unit length, kill vectors corr. to tiny
evalues
		 */
		numEigenVecs = 0;
		for(int i = 0; i < eigVectors.getColumnDimension(); i++) {
			Matrix tmp;
			if (values[i] < 0.0001)
			{
				tmp = new Matrix(eigVectors.getRowDimension(),1);
			}
			else
			{
				tmp = eigVectors.getMatrix(0,eigVectors.getRowDimension()-1,i,i).times(
						1 / eigVectors.getMatrix(0, eigVectors.getRowDimension() - 1, i,
i).normF());
				numEigenVecs++;
			}
			eigVectors.setMatrix(0,eigVectors.getRowDimension()-1,i,i,tmp);
			//eigVectors.timesEquals(1 / eigVectors.getMatrix(0,
eigVectors.getRowDimension() - 1, i, i).normInf());
		}
		eigVectors = eigVectors.getMatrix(0,eigVectors.getRowDimension() - 1, 0,
numEigenVecs - 1);
		
		trained = true;
        
		
		/*System.out.println("There are " + numGood + "
eigenVectors\n\nEigenVectorSize");
		System.out.println(eigVectors.getRowDimension());
		System.out.println(eigVectors.getColumnDimension());
		try {
            PrintWriter pw = new PrintWriter("c:\\tmp\\test.txt");
            eigVectors.print(pw, 8, 4);
            pw.flush();
            pw.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
        
		int width = pics[0].img.getWidth(null);
		BufferedImage biAvg = imageFromMatrix(bigAvg.getArrayCopy()[0], width);
		
		try {
            saveImage(new File("c:\\tmp\\test.jpg"), biAvg);
        } catch (IOException e1) {
            e1.printStackTrace();
        }*/
	}
	
	/**
	 * Returns a number of eigenFace values to be used in a feature space
	 * @param pic
	 * @param number number of eigen feature values.  
	 * @return will be of length number or this.getNumEigenVecs whichever is
the smaller
	 */
	public double[] getEigenFaces(Picture pic, int number)
	{
		if (number > numEigenVecs)		//adjust the number to the maxium number of
eigen vectors availiable
			number = numEigenVecs;
		
		double[] ret = new double[number];
		
		double[] pixels = pic.getImagePixels();
		Matrix face = new Matrix(pixels, pixels.length);
		Matrix Vecs = eigVectors.getMatrix(0,eigVectors.getRowDimension()-1, 0,
number-1).transpose();
		
		Matrix rslt = Vecs.times(face);
		
		for (int i=0; i<number; i++)
		{
			ret[i] = rslt.get(i,0);
		}
		
		return ret;
	}
	
	

	/**
	 * Gets the diagonal of a matrix
	 * @param M matrix
	 * @return
	 */
	private double[] diag(Matrix M) {
	    double[] dvec = new double[M.getColumnDimension()];
	    for(int i = 0; i < M.getColumnDimension(); i++)
	        dvec[i] = M.get(i, i);
	    return dvec;
	    
	}
	/**
	 * Sorts the Eigenvalues and vectors in decending order
	 * 
	 * @param D = eigen Values
	 * @param V = eigen Vectors
	 * @return
	 */
	private Matrix[] sortem(Matrix D, Matrix V) {
	    //dvec = diag(D); // get diagonal components
	    double[] dvec = diag(D);
	    		    
	    //NV = zeros(size(V));
	    
	    
	    //[dvec,index_dv] = sort(dvec); // sort dvec, maintain index in
index_dv
	    
	    class di_pair{ double value; int index; };
	    di_pair[] dvec_indexed = new di_pair[dvec.length];
	    for(int i = 0; i < dvec_indexed.length; i++) {
	        dvec_indexed[i] = new di_pair();
	        dvec_indexed[i].index = i;
	        dvec_indexed[i].value = dvec[i];
	    }
	        
	    Comparator di_pair_sort = new Comparator() {
            public int compare(Object arg0, Object arg1) {
                di_pair lt = (di_pair)arg0;
                di_pair rt = (di_pair)arg1;
                double dif = (lt.value - rt.value);
                if(dif > 0) return -1;
                if(dif < 0) return 1;
                else return 0;
            }
	    };
	    Arrays.sort(dvec_indexed, di_pair_sort);
	    
	    //index_dv = flipud(index_dv);
	    //for i = 1:size(D,1)
	    //  ND(i,i) = D(index_dv(i),index_dv(i));
	    //  NV(:,i) = V(:,index_dv(i));
	    //end;

	    Matrix D2 = new Matrix(D.getRowDimension(), D.getColumnDimension());
	    Matrix V2 = new Matrix(V.getRowDimension(), V.getColumnDimension());
	    
	    for(int i = 0; i < dvec_indexed.length; i++) {
	        D2.set(i, i, D.get(dvec_indexed[i].index, dvec_indexed[i].index));
	        int height = V.getRowDimension() - 1;
	        Matrix tmp =
V.getMatrix(dvec_indexed[i].index,dvec_indexed[i].index,0,height);
	        V2.setMatrix(i, i,0,height, tmp);
	    }
	    //TODO : Not sure why, but this has to be flipped - check this out
maybe?
	    Matrix V3 = new Matrix(V.getRowDimension(), V.getColumnDimension());
	    for (int i=0; i<V3.getRowDimension(); i++)
	    {
	    	for (int j=0; j< V3.getColumnDimension(); j++)
	    	{
	    		V3.set(i,j,V2.get(V3.getRowDimension() - i - 1,
V3.getColumnDimension() - j - 1));
	    	}
	    }
	    
	    return new Matrix[] { D2, V3 };
	}
	
	
	public boolean isTrained() {
		return trained;
	}
	public int getNumEigenVecs() {
		return numEigenVecs;
	}
}


I am not sure this code easily can be parallel computed,or how to change
this code to add the parallel  compuation.any advice will  be
appreciated.thanks in advance.

-- 
View this message in context: http://www.nabble.com/Hama--Problem-tp23630189p23630189.html
Sent from the Hadoop core-user mailing list archive at Nabble.com.


Mime
View raw message