eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yonzhang2...@apache.org
Subject [4/7] incubator-eagle git commit: EAGLE-334 clean user profile code clean user profile code in Eagle 0.5 and redesign in Eagle 0.6
Date Fri, 10 Jun 2016 01:29:29 GMT
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml
b/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml
deleted file mode 100644
index 25d079c..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/assembly/assembly.xml
+++ /dev/null
@@ -1,67 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~    http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>assembly</id>
-    <formats>
-        <format>jar</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>.</outputDirectory>
-            <useProjectArtifact>false</useProjectArtifact>
-            <unpack>true</unpack>
-            <scope>runtime</scope>
-            <unpackOptions>
-                <excludes>
-                    <exclude>log4j.properties</exclude>
-                </excludes>
-            </unpackOptions>
-            <excludes>
-                <!--<exclude>org.apache.spark:*</exclude>-->
-                <exclude>org.apache.storm:*</exclude>
-                <exclude>org.clojure:*</exclude>
-                <exclude>org.scala-lang:*</exclude>
-                <exclude>org.apache.commons:math3</exclude>
-                <exclude>com.fasterxml.jackson.core:*</exclude>
-                <exclude>com.fasterxml.jackson.module:*</exclude>
-                <exclude>org.slf4j:slf4j-api:*</exclude>
-                <exclude>org.slf4j:slf4j-log4j12:*</exclude>
-                <exclude>log4j:log4j:*</exclude>
-                <exclude>org.slf4j:log4j-over-slf4j:*</exclude>
-                <exclude>org.apache.hadoop:hadoop-yarn-*</exclude>
-                <exclude>org.apache.hadoop:hadoop-mapreduce-*</exclude>
-                <!--<exclude>org.apache.hbase:*</exclude>-->
-            </excludes>
-        </dependencySet>
-    </dependencySets>
-    <fileSets>
-        <fileSet>
-            <directory>${project.build.outputDirectory}</directory>
-            <outputDirectory>.</outputDirectory>
-            <excludes>
-                <exclude>log4j.properties</exclude>
-                <exclude>application.conf</exclude>
-                <exclude>hdfs-audit.log</exclude>
-            </excludes>
-        </fileSet>
-    </fileSets>
-</assembly>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java
b/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java
deleted file mode 100644
index 909b7b7..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/JavaUserProfileModeler.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * 
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-package org.apache.eagle.security.userprofile.model;
-
-import org.apache.commons.math3.linear.RealMatrix;
-import scala.collection.JavaConversions;
-import scala.collection.immutable.List;
-
-/**
- * @since 10/21/15
- */
-public abstract class JavaUserProfileModeler<M,C extends UserProfileContext> implements
UserProfileModeler<M,C> {
-    @Override
-    public List<M> build(String site, String user, RealMatrix matrix) {
-        java.util.List<M> models = generate(site, user, matrix);
-        if (models != null) {
-            return JavaConversions.asScalaBuffer(models).toList();
-        } else {
-            return null;
-        }
-    }
-
-    /**
-     * Java delegate method for {@code eagle.security.userprofile.model.JavaUserProfileModeler#build(String
site, String user, RealMatrix matrix)}
-     *
-     * @param site eagle site
-     * @param user eagle user
-     * @param matrix user activity matrix
-     * @return Generate user profile model
-     */
-    public abstract java.util.List<M> generate(String site, String user, RealMatrix
matrix);
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java
b/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java
deleted file mode 100644
index cb9d660..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/eigen/UserProfileEigenModeler.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model.eigen;
-
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.commons.math3.exception.NotStrictlyPositiveException;
-import org.apache.commons.math3.linear.*;
-import org.apache.commons.math3.stat.correlation.Covariance;
-import org.apache.commons.math3.stat.descriptive.moment.Mean;
-import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
-import org.apache.eagle.security.userprofile.model.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.List;
-
-public class UserProfileEigenModeler extends JavaUserProfileModeler<UserProfileEigenModel,UserProfileContext>
{
-
-    private final String[] cmdTypes;
-    private UserCommandStatistics[] statistics;
-    private String outputLocation;
-    private RealMatrix finalMatrixWithoutLowVariantCmds;
-    private RealMatrix covarianceMatrix;
-    private RealMatrix uMatrix;
-    private RealMatrix vMatrix;
-    private RealMatrix diagonalMatrix;
-    private int dimension =0;
-    private static final double MAINTENED_VARIANCE = 0.99;
-    private RealVector[] principalComponents;
-    private RealVector maximumL2Norm; //Double.NEGATIVE_INFINITY;
-    private RealVector minimumL2Norm; //= Double.POSITIVE_INFINITY;
-    private RealVector minVector;
-    private RealVector maxVector;
-
-    private final static Logger LOG = LoggerFactory.getLogger(UserProfileEigenModeler.class);
-    private static final double lowVarianceVal= 0.001;
-
-    public RealVector[] getPrincipalComponents() {
-        return principalComponents;
-    }
-
-    public void setPrincipalComponents(RealVector[] principalComponents) {
-        this.principalComponents = principalComponents;
-    }
-
-    public UserProfileEigenModeler(String[] cmdTypes){
-        this.cmdTypes = cmdTypes;
-    }
-
-    public UserProfileEigenModeler(){
-        this.cmdTypes = UserProfileConstants.DEFAULT_CMD_TYPES;
-    }
-
-    public RealVector getMaximumL2Norm() {
-        return maximumL2Norm;
-    }
-
-    public void setMaximumL2Norm(RealVector maximumL2Norm) {
-        this.maximumL2Norm = maximumL2Norm;
-    }
-
-    public RealVector getMinimumL2Norm() {
-        return minimumL2Norm;
-    }
-
-    public void setMinimumL2Norm(RealVector minimumL2Norm) {
-        this.minimumL2Norm = minimumL2Norm;
-    }
-
-    public RealVector getMinVector() {
-        return minVector;
-    }
-
-    public void setMinVector(RealVector minVector) {
-        this.minVector = minVector;
-    }
-
-    public RealVector getMaxVector() {
-        return maxVector;
-    }
-
-    public void setMaxVector(RealVector maxVector) {
-        this.maxVector = maxVector;
-    }
-
-    public UserCommandStatistics[] getStatistics() {
-        return statistics;
-    }
-
-    public void setStatistics(UserCommandStatistics[] statistics) {
-        this.statistics = statistics;
-    }
-
-    public String getOutputLocation() {
-        return outputLocation;
-    }
-
-    public void setOutputLocation(String outputLocation) {
-        this.outputLocation = outputLocation;
-    }
-
-    private void computeStats(RealMatrix m){
-
-        if(m.getColumnDimension() != this.cmdTypes.length){
-            LOG.error("Please fix the commands list in config file");
-            return;
-        }
-        statistics = new UserCommandStatistics[m.getColumnDimension()];
-        for(int i=0; i<m.getColumnDimension(); i++){
-            UserCommandStatistics stats = new UserCommandStatistics();
-            stats.setCommandName(this.cmdTypes[i]);
-            RealVector colData = m.getColumnVector(i);
-            StandardDeviation deviation = new StandardDeviation();
-            double stddev = deviation.evaluate(colData.toArray());
-            //LOG.info("stddev is nan?" + (stddev == Double.NaN? "yes":"no"));
-            if(stddev <= lowVarianceVal)
-                stats.setLowVariant(true);
-            else
-                stats.setLowVariant(false);
-            stats.setStddev(stddev);
-            Mean mean = new Mean();
-            double mu = mean.evaluate(colData.toArray());
-            //LOG.info("mu is nan?" + (mu == Double.NaN? "yes":"no"));
-            stats.setMean(mu);
-            statistics[i] = stats;
-        }
-    }
-
-    private RealMatrix normalizeData(RealMatrix matrix){
-
-        RealMatrix normalizedData = new Array2DRowRealMatrix(matrix.getRowDimension(), matrix.getColumnDimension());
-        for(int i=0; i< matrix.getRowDimension(); i++){
-            for(int j=0; j < matrix.getColumnDimension(); j++){
-                // TODO:  statistics[j].getStddev() == 0 what should the value be if stddev
is o?
-                double value = statistics[j].getStddev() == 0? 0 : (matrix.getEntry(i, j)
- statistics[j].getMean())/statistics[j].getStddev();
-                normalizedData.setEntry(i, j, value);
-            }
-        }
-        return normalizedData;
-    }
-
-    private void computeCovarianceAndSVD(RealMatrix inputMat, int containsLowVariantCol){
-
-        int finalMatrixRow=0;
-        int finalMatrixCol=0;
-
-        LOG.info("containsLowVariantCol size: " + containsLowVariantCol);
-        int colDimension = (inputMat.getColumnDimension() - containsLowVariantCol);
-        try {
-            finalMatrixWithoutLowVariantCmds = new Array2DRowRealMatrix(inputMat.getRowDimension(),
colDimension);
-        }catch (NotStrictlyPositiveException e){
-            LOG.error(String.format("Failed to build matrix [rowDimension:%s, columnDimension:
%s]",inputMat.getRowDimension(),colDimension),e);
-            throw e;
-        }
-
-        for(int i=0; i < inputMat.getRowDimension(); i++){
-            for(int j=0; j < inputMat.getColumnDimension();j++){
-                if(!statistics[j].isLowVariant()){
-                    finalMatrixWithoutLowVariantCmds.setEntry(finalMatrixRow, finalMatrixCol,
inputMat.getEntry(i, j));
-                    finalMatrixCol++;
-                }
-            }
-            finalMatrixCol = 0;
-            finalMatrixRow++;
-        }
-
-        Covariance cov;
-        try {
-            cov = new Covariance(finalMatrixWithoutLowVariantCmds.getData());
-        }catch (Exception ex){
-            throw new IllegalArgumentException(String.format("Failed to create covariance
from matrix [ %s x %s ]",finalMatrixWithoutLowVariantCmds.getRowDimension(),finalMatrixWithoutLowVariantCmds.getColumnDimension()),ex);
-        }
-        covarianceMatrix = cov.getCovarianceMatrix();
-        SingularValueDecomposition svd = new SingularValueDecomposition(covarianceMatrix);
-        diagonalMatrix = svd.getS();
-        uMatrix = svd.getU();
-        vMatrix = svd.getV();
-    }
-
-    private void computeDimensionWithMaxVariance(){
-        if(diagonalMatrix == null || diagonalMatrix.getRowDimension() != diagonalMatrix.getColumnDimension()){
-            LOG.error("Diagonal matrix is not correctly computed, SVD should have been done");
-            return;
-        }
-
-        double sumTotal = 0.0;
-        for(int i=0; i < diagonalMatrix.getRowDimension(); i++){
-            sumTotal += diagonalMatrix.getEntry(i, i);
-        }
-        //LOG.info("sum total: " + sumTotal);
-        double sumEach = 0.0;
-        for(int i=0; i < diagonalMatrix.getRowDimension(); i++){
-            sumEach += diagonalMatrix.getEntry(i, i);
-            if((sumEach/sumTotal) >= MAINTENED_VARIANCE){
-                dimension = i;
-                break;
-            }
-
-        }
-        if(dimension > finalMatrixWithoutLowVariantCmds.getColumnDimension())
-            LOG.error("Reduced dimension cannot be smaller than original size of the feature");
-
-    }
-
-    private void computePrincipalComponents(){
-        principalComponents = new ArrayRealVector[uMatrix.getColumnDimension()];
-        for(int i=0; i < uMatrix.getColumnDimension(); i++){
-            principalComponents[i] = uMatrix.getColumnVector(i).mapMultiply(Math.sqrt(diagonalMatrix.getEntry(i,
i)));
-        }
-    }
-
-    private RealMatrix computeMaxDistanceOnPCs(int index){
-
-        RealMatrix pc = new Array2DRowRealMatrix(principalComponents[index].toArray());
-        RealMatrix transposePC1 = pc.transpose();
-        RealMatrix finalDataTranspose = finalMatrixWithoutLowVariantCmds.transpose();
-
-        RealMatrix trainingData = pc.multiply(transposePC1).multiply(finalDataTranspose);
-        RealMatrix trainingDataTranspose = trainingData.transpose();
-
-        double maxDistance = 0.0;
-        double minDistance = 0.0;
-        RealVector p1 = null, p2 = null;
-        if(LOG.isDebugEnabled()) LOG.debug("Training data transpose size: " + trainingDataTranspose.getRowDimension()
+ "x" + trainingDataTranspose.getColumnDimension());
-        for(int i = 0; i < trainingDataTranspose.getRowDimension(); i++){
-            RealVector iRowVector = new ArrayRealVector(trainingDataTranspose.getRow(i));
-            RealVector transposePC1Vect = transposePC1.getRowVector(0);
-            double distance = iRowVector.getDistance(transposePC1Vect);
-            if(distance > maxDistance){
-                maxDistance = distance;
-                p1 = iRowVector;
-                p2 = transposePC1Vect;
-            }
-            if(distance < minDistance)
-                minDistance = distance;
-            //}
-        }
-        maximumL2Norm.setEntry(index, maxDistance);
-        minimumL2Norm.setEntry(index, minDistance);
-        minVector = p1;
-        maxVector = p2;
-        return trainingDataTranspose;
-    }
-
-    @Override
-    public List<UserProfileEigenModel> generate(String site, String user, RealMatrix
matrix) {
-        LOG.info(String.format("Receive aggregated user activity matrix: %s size: %s x %s",user,matrix.getRowDimension(),matrix.getColumnDimension()));
-        computeStats(matrix);
-        RealMatrix normalizedInputMatrix = normalizeData(matrix);
-        int lowVariantColumnCount = 0;
-        for(int j=0; j < normalizedInputMatrix.getColumnDimension();j++){
-            if(statistics[j].isLowVariant()){
-                lowVariantColumnCount++;
-            }
-        }
-
-        if(normalizedInputMatrix.getColumnDimension() == lowVariantColumnCount){
-            LOG.info("found user: " + user + " with all features being low variant. Nothing
to do...");
-            UserProfileEigenModel noopModel = new UserProfileEigenModel(System.currentTimeMillis(),site,user,null,null,0,null,null,null,null,null,statistics);
-            return Arrays.asList(noopModel);
-        }
-        else {
-            computeCovarianceAndSVD(normalizedInputMatrix, lowVariantColumnCount);
-            computeDimensionWithMaxVariance();
-            computePrincipalComponents();
-            maximumL2Norm = new ArrayRealVector(principalComponents.length);
-            minimumL2Norm = new ArrayRealVector(principalComponents.length);
-
-            for (int i = 0; i < principalComponents.length; i++) {
-                RealMatrix trainingDataTranspose = computeMaxDistanceOnPCs(i);
-            }
-
-            UserProfileEigenModel userprofileEigenModel = new UserProfileEigenModel(System.currentTimeMillis(),site,user,
uMatrix, diagonalMatrix, dimension, minVector, maxVector, principalComponents, maximumL2Norm,
minimumL2Norm, statistics);
-            return Arrays.asList(userprofileEigenModel);
-        }
-    }
-
-    @Override
-    public UserProfileContext context() {
-        return new UserProfileContext(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java
b/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java
deleted file mode 100644
index 1b24137..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/java/org/apache/eagle/security/userprofile/model/kde/UserProfileKDEModeler.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.model.kde;
-
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.commons.math3.linear.RealVector;
-import org.apache.commons.math3.stat.descriptive.moment.Mean;
-import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
-import org.apache.eagle.security.userprofile.model.JavaUserProfileModeler;
-import org.apache.eagle.security.userprofile.model.UserCommandStatistics;
-import org.apache.eagle.security.userprofile.model.UserProfileContext;
-import org.apache.eagle.security.userprofile.model.UserProfileKDEModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-public class UserProfileKDEModeler extends JavaUserProfileModeler<UserProfileKDEModel,UserProfileContext>
{
-
-    private final String[] cmdTypes;
-    private UserCommandStatistics[] statistics;
-    private RealMatrix finalMatrixWithoutLowVariantCmds;
-    private RealMatrix covarianceMatrix;
-    private double[] probabilityEstimation;
-    private double minProbabilityEstimate;
-    private double maxProbabilityEstimate;
-    private double nintyFivePercentileEstimate;
-    private double medianProbabilityEstimate;
-    private final static Logger LOG = LoggerFactory.getLogger(UserProfileKDEModeler.class);
-    private static final double lowVarianceVal= 0.001;
-
-    public UserProfileKDEModeler(String[] cmdTypes){
-        this.cmdTypes = cmdTypes;
-    }
-
-    public UserProfileKDEModeler(){
-        this.cmdTypes = UserProfileConstants.DEFAULT_CMD_TYPES;
-    }
-
-    private void computeStats(RealMatrix m){
-        if(m.getColumnDimension() !=  this.cmdTypes.length){
-            LOG.error("Please fix the commands list in config file");
-        }
-
-        statistics = new UserCommandStatistics[m.getColumnDimension()];
-
-        for(int i=0; i<m.getColumnDimension(); i++){
-            UserCommandStatistics stats = new UserCommandStatistics();
-            stats.setCommandName(this.cmdTypes[i]);
-            RealVector colData = m.getColumnVector(i);
-            StandardDeviation deviation = new StandardDeviation();
-            double stddev = deviation.evaluate(colData.toArray());
-
-            if(LOG.isDebugEnabled()) LOG.debug("Stddev is NAN ? " + (Double.isNaN(stddev)
? "yes" : "no"));
-            if(stddev <= lowVarianceVal)
-                stats.setLowVariant(true);
-            else
-                stats.setLowVariant(false);
-
-            stats.setStddev(stddev);
-            Mean mean = new Mean();
-            double mu = mean.evaluate(colData.toArray());
-            if(LOG.isDebugEnabled()) LOG.debug("mu is NAN ? " + (Double.isNaN(mu)? "yes":"no"));
-
-            stats.setMean(mu);
-            statistics[i]=stats;
-        }
-    }
-
-    private void computeProbabilityDensityEstimation(RealMatrix inputMat){
-
-        probabilityEstimation = new double[inputMat.getRowDimension()];
-        for(int i=0; i < probabilityEstimation.length; i++)
-            probabilityEstimation[i] = 1.0;
-
-        for(int i=0; i < inputMat.getRowDimension(); i++){
-            for(int j=0; j < inputMat.getColumnDimension(); j++){
-                if(statistics[j].getStddev() > 0){
-                    double stddev = statistics[j].getStddev();
-                    double mean = statistics[j].getMean();
-                    double sqrt2PI = Math.sqrt(2.0*Math.PI);
-                    double denominatorFirstPart = sqrt2PI*stddev;
-                    double squareMeanNormal = Math.pow((inputMat.getEntry(i, j) - mean),
2);
-                    double twoPowStandardDev = Math.pow(stddev, 2);
-                    double twoTimesTwoPowStandardDev = 2.0*twoPowStandardDev;
-                    probabilityEstimation[i] *= ((1.00/denominatorFirstPart)
-                            *(Math.exp(-(squareMeanNormal/twoTimesTwoPowStandardDev))));
-                }
-            }
-        }
-
-        java.util.List<Double> listProb = new ArrayList<Double>();
-        for(int i=0; i < probabilityEstimation.length; i++){
-            probabilityEstimation[i] = Math.log10(probabilityEstimation[i]);
-            listProb.add(probabilityEstimation[i]);
-        }
-
-        Collections.sort(listProb);
-        int i=0;
-        for(double d:listProb)
-            probabilityEstimation[i++]=d;
-
-        minProbabilityEstimate = probabilityEstimation[probabilityEstimation.length -1];
-        maxProbabilityEstimate = probabilityEstimation[0];
-
-        int len = probabilityEstimation.length;
-        int nintyFivePercentIndex = (int)Math.round(0.05*len);
-        int medianPercentIndex = (int)Math.round(0.5*len);
-        if(medianPercentIndex >= len)
-            medianProbabilityEstimate = probabilityEstimation[medianPercentIndex-1];
-        else
-            medianProbabilityEstimate = probabilityEstimation[medianPercentIndex];
-        nintyFivePercentileEstimate = probabilityEstimation[nintyFivePercentIndex];
-    }
-
-    @Override
-    public List<UserProfileKDEModel> generate(String site, String user, RealMatrix
matrix) {
-        LOG.info(String.format("Receive aggregated user activity matrix: %s size: %s x %s",user,matrix.getRowDimension(),matrix.getColumnDimension()));
-        computeStats(matrix);
-        computeProbabilityDensityEstimation(matrix);
-        UserProfileKDEModel userprofileKDEModel = new UserProfileKDEModel(System.currentTimeMillis(),site,user,
statistics, minProbabilityEstimate, maxProbabilityEstimate, nintyFivePercentileEstimate, medianProbabilityEstimate);
-        return Arrays.asList(userprofileKDEModel);
-    }
-
-    @Override
-    public UserProfileContext context() {
-        return new UserProfileContext(UserProfileConstants.KDE_ALGORITHM);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf
b/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf
deleted file mode 100644
index db896d0..0000000
--- a/eagle-security/eagle-security-userprofile/training/src/main/resources/application.conf
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-eagle{
-	site = "sandbox"
-
-	service{
-		host = "localhost"
-		port = 9098
-    username = "admin"
-    password = "secret"
-	}
-
-	userprofile {
-		jar="/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/target/eagle-security-userprofile-training-0.1.0-assembly.jar"
-		driver-shell="spark-submit"
-		driver-classpath="/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/target/lib/commons-math3-3.5.jar"
-		//	model-audit-path = "/tmp/audit/${yyyy-MM-*}/*.tar.gz"
-		//	agg-audit-path = "/tmp/audit/${yyyy-MM-dd}/${yyyy-MM-dd-hh}.tar.gz"
-
-		training-audit-path = "/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/src/main/resources/hdfs-audit.log"
-		detection-audit-path = "/Users/path/Workspace/eagle-parent/eagle/eagle-security/eagle-security-userprofile/training/src/main/resources/hdfs-audit.log"
-	}
-}
\ No newline at end of file


Mime
View raw message