eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yonzhang2...@apache.org
Subject [5/7] incubator-eagle git commit: EAGLE-334 clean user profile code clean user profile code in Eagle 0.5 and redesign in Eagle 0.6
Date Fri, 10 Jun 2016 01:29:30 GMT
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserActivityAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserActivityAggregatorImpl.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserActivityAggregatorImpl.java
deleted file mode 100644
index 52bdbf0..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserActivityAggregatorImpl.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.impl;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.eagle.common.DateTimeUtil;
-import org.apache.eagle.datastream.Collector;
-import org.apache.eagle.security.userprofile.TimeWindow;
-import org.apache.eagle.security.userprofile.UserActivityAggregator;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.eagle.security.userprofile.UserProfileUtils;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModelEntity;
-import org.joda.time.Period;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.Tuple2;
-
-import java.util.*;
-
-/**
- * UserActivityAggregator implementation
- *
- * @see org.apache.eagle.security.userprofile.UserActivityAggregator
- * @since 9/28/15
- */
-public class UserActivityAggregatorImpl implements UserActivityAggregator {
-    private final Map<Long,TimeWindow> timeWindowCache = Collections.synchronizedMap(new HashMap<Long, TimeWindow>());
-    private final Map<TimeWindow,Map<Tuple2<String,String>,Double>> counterCache = Collections.synchronizedMap(new HashMap<TimeWindow, Map<Tuple2<String, String>, Double>>());
-    private final static Logger LOG = LoggerFactory.getLogger(UserActivityAggregatorImpl.class);
-    private final String site;
-    private final long safeWindowMs;
-    private final Period granularity;
-    private final List<String> cmdFeatures;
-
-    public UserActivityAggregatorImpl(List<String> cmdFeatures,Period granularity,String site,long safeWindowMs){
-        this.cmdFeatures = cmdFeatures;
-        this.granularity = granularity;
-        this.site = site;
-        this.safeWindowMs = safeWindowMs;
-    }
-
-    /**
-     * Build new time window
-     *
-     * @param from where time window start from
-     * @param to where time window end at
-     * @return new time window instance
-     */
-    private TimeWindow newTimeWindowInstance(Long from, Long to){
-        return new TimeWindowImpl(from,to,this.safeWindowMs);
-    }
-
-    @Override
-    public void accumulate(Map<String, Object> event, Collector<Tuple2<String, UserActivityAggModelEntity>> collector) {
-        Long timestamp = (Long) event.get("timestamp");
-        Long baseTimestamp = UserProfileUtils.formatMillisecondsByPeriod(timestamp, this.granularity);
-
-        TimeWindow window = timeWindowCache.get(baseTimestamp);
-        if(window == null) {
-            window = newTimeWindowInstance(baseTimestamp, baseTimestamp + this.granularity.toStandardSeconds().getSeconds() * 1000);
-            LOG.info(String.format("Creating %s",window.toString()));
-            // Initialize time window
-            init(window);
-        }
-        List<TimeWindow> expiredWindows = new ArrayList<>();
-        for(TimeWindow win:timeWindowCache.values()){
-            if(win.accept(timestamp)) accumulate(win, event);
-            if(win.expire()){
-                // Flush time window
-                flush(win, collector);
-                expiredWindows.add(win);
-            }
-        }
-
-        // clean expired time window
-        for(TimeWindow win:expiredWindows){
-            if(LOG.isDebugEnabled()) LOG.debug(String.format("%s is expired at [%s]", win.toString(), DateTimeUtil.millisecondsToHumanDateWithMilliseconds(timestamp)));
-            close(win);
-        }
-    }
-
-    private void close(TimeWindow win) {
-        counterCache.remove(win);
-        timeWindowCache.remove(win.from());
-    }
-
-    private void init(TimeWindow window) {
-        timeWindowCache.put(window.from(),window);
-        Map<Tuple2<String,String>,Double> counter = counterCache.get(window);
-        if(counter == null){
-            counter = new HashMap<>();
-            counterCache.put(window,counter);
-        }
-    }
-
-    private void accumulate(TimeWindow window, Map<String, Object> event) {
-        Map<Tuple2<String,String>,Double> counter = counterCache.get(window);
-        String user = (String) event.get("user");
-        String cmd = (String) event.get("cmd");
-        Tuple2<String,String> tuple = new Tuple2<>(user,cmd);
-        if(counter.containsKey(tuple)){
-            Double count = counter.get(tuple) + 1;
-            counter.put(tuple,count);
-        }else{
-            counter.put(tuple,1.0);
-        }
-    }
-
-    private void flush(TimeWindow window, Collector<Tuple2<String, UserActivityAggModelEntity>> collector){
-        Map<Tuple2<String,String>,Double> counter = counterCache.get(window);
-
-        Map<String,Map<String,Double>> tmp = new HashMap<>();
-        for(Map.Entry<Tuple2<String,String>,Double> entry:counter.entrySet()){
-            String user = entry.getKey()._1();
-            Map<String,Double> cmdCount = tmp.get(user);
-            if(cmdCount == null) cmdCount = new HashMap<>();
-            cmdCount.put(entry.getKey()._2(),entry.getValue());
-            tmp.put(user,cmdCount);
-        }
-
-        for(Map.Entry<String,Map<String,Double>> entry:tmp.entrySet()){
-            final String user = entry.getKey();
-            final Map<String,Double> cmdCountMap = entry.getValue();
-
-            UserActivityAggModelEntity model = new UserActivityAggModelEntity();
-            model.setTags(new HashMap<String, String>(){{
-                put(UserProfileConstants.SITE_TAG,site);
-                put(UserProfileConstants.USER_TAG,user);
-            }});
-
-            // Build matrix
-            double[][] matrix = new double[1][this.cmdFeatures.size()];
-
-            for(int i=0;i<cmdFeatures.size();i++){
-                Double cnt = cmdCountMap.get(cmdFeatures.get(i));
-                if(cnt == null) cnt = 0.0;
-                matrix[0][i] = cnt;
-            }
-            model.setCmdTypes(cmdFeatures);
-            model.setCmdMatrix(matrix);
-            model.setTimestamp(window.to());
-
-            if(LOG.isDebugEnabled()) {
-                try {
-                    LOG.debug(new ObjectMapper().writeValueAsString(model));
-                } catch (JsonProcessingException e) {
-                    LOG.error(e.getMessage(), e);
-                }
-            }
-            collector.collect(new Tuple2(user, model));
-        }
-        LOG.info(String.format("Flushed %s records during %s", tmp.size(), window));
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyEigenEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyEigenEvaluator.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyEigenEvaluator.java
deleted file mode 100644
index db7897f..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyEigenEvaluator.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.impl;
-
-import org.apache.eagle.ml.model.MLCallbackResult;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModel;
-import org.apache.eagle.security.userprofile.model.UserCommandStatistics;
-import org.apache.eagle.security.userprofile.model.UserProfileEigenModel;
-import org.apache.commons.math3.linear.Array2DRowRealMatrix;
-import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.commons.math3.linear.RealVector;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.eagle.security.userprofile.model.UserProfileEigenModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class UserProfileAnomalyEigenEvaluator extends AbstractUserProfileEigenEvaluator {
-    private static Logger LOG = LoggerFactory.getLogger(UserProfileAnomalyEigenEvaluator.class);
-    private RealMatrix normalizeData(RealMatrix matrix, UserProfileEigenModel model){
-        RealMatrix normalizedData = new Array2DRowRealMatrix(matrix.getRowDimension(), matrix.getColumnDimension());
-        if(LOG.isDebugEnabled()) LOG.debug("model statistics size: " + model.statistics().length);
-        for(int i=0; i< matrix.getRowDimension(); i++){
-            for(int j=0; j < matrix.getColumnDimension(); j++){
-                double value = (matrix.getEntry(i, j) - model.statistics()[j].getMean())/model.statistics()[j].getStddev();
-                normalizedData.setEntry(i, j, value);
-            }
-        }
-        return normalizedData;
-    }
-
-    @Override
-    public List<MLCallbackResult> detect(final String user, final String algorithm, UserActivityAggModel userActivity, UserProfileEigenModel aModel) {
-        RealMatrix inputData = userActivity.matrix();
-        LOG.warn("EigenBasedAnomalyDetection predictAnomaly called with dimension: " + inputData.getRowDimension() + "x" + inputData.getColumnDimension());
-
-        if(aModel== null){
-            LOG.warn("nothing to do as the input model does not have required values, returning from evaluating this algorithm..");
-            return null;
-        }
-
-        List<MLCallbackResult> mlCallbackResults = new ArrayList<MLCallbackResult>();
-        RealMatrix normalizedMat = normalizeData(inputData, aModel);
-
-        UserCommandStatistics[] listStats = aModel.statistics();
-        int colWithHighVariant = 0;
-
-        for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
-            if (listStats[j].isLowVariant() == false) {
-                colWithHighVariant++;
-            }
-        }
-
-        final Map<String,String> context = new HashMap<String, String>(){{
-            put(UserProfileConstants.USER_TAG,user);
-            put(UserProfileConstants.ALGORITHM_TAG,algorithm);
-        }};
-
-        Map<Integer, String> lineNoWithVariantBasedAnomalyDetection = new HashMap<Integer, String>();
-        for (int i = 0; i < normalizedMat.getRowDimension(); i++) {
-            MLCallbackResult aResult = new MLCallbackResult();
-            aResult.setAnomaly(false);
-            aResult.setContext(context);
-
-            for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
-                //LOG.info("mean for j=" + j + " is:" + listStats[j].getMean());
-                //LOG.info("stddev for j=" + j + " is:" + listStats[j].getStddev());
-                if (listStats[j].isLowVariant() == true) {
-                    //LOG.info(listOfCmds[j] + " is low variant");
-                    if (normalizedMat.getEntry(i, j) > listStats[j].getMean()) {
-                        lineNoWithVariantBasedAnomalyDetection.put(i, "lowVariantAnomaly");
-                        aResult.setAnomaly(true);
-                        aResult.setTimestamp(userActivity.timestamp());
-                        aResult.setFeature(listStats[j].getCommandName());
-                        aResult.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
-                        List<String> datapoints = new ArrayList<String>();
-                        double[] rowVals = inputData.getRow(i);
-                        for(double rowVal:rowVals)
-                            datapoints.add(rowVal+"");
-                        aResult.setDatapoints(datapoints);
-                        aResult.setId(user);
-                        //mlCallbackResults.add(aResult);
-                    } /*else {
-                        aResult.setAnomaly(false);
-                        aResult.setTimestamp(userActivity.timestamp());
-                        mlCallbackResults.add(aResult);
-                    }*/
-                }
-            }
-            mlCallbackResults.add(i, aResult);
-            //return results;
-        }
-
-        //LOG.info("results size here: " + results.length);
-
-        //LOG.info("col with high variant: " + colWithHighVariant);
-        RealMatrix finalMatWithoutLowVariantFeatures = new Array2DRowRealMatrix(normalizedMat.getRowDimension(), colWithHighVariant);
-        //LOG.info("size of final test data: " + finalMatWithoutLowVariantFeatures.getRowDimension() +"x"+ finalMatWithoutLowVariantFeatures.getColumnDimension());
-        int finalMatrixRow = 0;
-        int finalMatrixCol = 0;
-        for (int i = 0; i < normalizedMat.getRowDimension(); i++) {
-            for (int j = 0; j < normalizedMat.getColumnDimension(); j++) {
-                if (listStats[j].isLowVariant() == false) {
-                    finalMatWithoutLowVariantFeatures.setEntry(finalMatrixRow, finalMatrixCol, normalizedMat.getEntry(i, j));
-                    finalMatrixCol++;
-                }
-            }
-            finalMatrixCol = 0;
-            finalMatrixRow++;
-        }
-        RealVector[] pcs = aModel.principalComponents();
-        //LOG.info("pc size: " + pcs.getRowDimension() +"x" + pcs.getColumnDimension());
-
-        RealMatrix finalInputMatTranspose = finalMatWithoutLowVariantFeatures.transpose();
-
-        for (int i = 0; i < finalMatWithoutLowVariantFeatures.getRowDimension(); i++) {
-            if (lineNoWithVariantBasedAnomalyDetection.get(i) == null) {
-                //MLCallbackResult result = new MLCallbackResult();
-                MLCallbackResult result = mlCallbackResults.get(i);
-                //result.setContext(context);
-                for (int sz = 0; sz < pcs.length; sz++) {
-                    double[] pc1 = pcs[sz].toArray();
-                    RealMatrix pc1Mat = new Array2DRowRealMatrix(pc1);
-                    RealMatrix transposePC1Mat = pc1Mat.transpose();
-                    RealMatrix testData = pc1Mat.multiply(transposePC1Mat).multiply(finalInputMatTranspose.getColumnMatrix(i));
-                    //LOG.info("testData size: " + testData.getRowDimension() + "x" + testData.getColumnDimension());
-                    RealMatrix testDataTranspose = testData.transpose();
-                    //LOG.info("testData transpose size: " + testDataTranspose.getRowDimension() + "x" + testDataTranspose.getColumnDimension());
-                    RealVector iRowVector = testDataTranspose.getRowVector(0);
-                    //RealVector pc1Vector = transposePC1Mat.getRowVector(sz);
-                    RealVector pc1Vector = transposePC1Mat.getRowVector(0);
-                    double distanceiRowAndPC1 = iRowVector.getDistance(pc1Vector);
-                    //LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " + model.getMaxL2Norm().getEntry(sz));
-                    //LOG.info("model.getMaxL2Norm().getEntry(sz):" + model.getMaxL2Norm().getEntry(sz));
-                    if (distanceiRowAndPC1 > aModel.maximumL2Norm().getEntry(sz)) {
-                        //LOG.info("distance from pc sz: " + sz + " " + distanceiRowAndPC1 + " " + model.getMaxL2Norm().getEntry(sz));
-                        result.setAnomaly(true);
-                        result.setFeature(aModel.statistics()[sz].getCommandName());
-                        result.setTimestamp(System.currentTimeMillis());
-                        result.setAlgorithm(UserProfileConstants.EIGEN_DECOMPOSITION_ALGORITHM);
-                        List<String> datapoints = new ArrayList<String>();
-                        double[] rowVals = inputData.getRow(i);
-                        for(double rowVal:rowVals)
-                            datapoints.add(rowVal+"");
-                        result.setDatapoints(datapoints);
-                        result.setId(user);
-                    }
-                }
-                mlCallbackResults.set(i, result);
-            }
-        }
-        return mlCallbackResults;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyKDEEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyKDEEvaluator.java b/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyKDEEvaluator.java
deleted file mode 100644
index 7e89b28..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/java/org/apache/eagle/security/userprofile/impl/UserProfileAnomalyKDEEvaluator.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.userprofile.impl;
-
-import org.apache.eagle.ml.model.MLCallbackResult;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModel;
-import org.apache.eagle.security.userprofile.model.UserCommandStatistics;
-import org.apache.eagle.security.userprofile.model.UserProfileKDEModel;
-import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.eagle.security.userprofile.UserProfileConstants;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModel;
-import org.apache.eagle.security.userprofile.model.UserCommandStatistics;
-import org.apache.eagle.security.userprofile.model.UserProfileKDEModel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.collection.JavaConversions;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class UserProfileAnomalyKDEEvaluator extends AbstractUserProfileKDEEvaluator{
-    private static Logger LOG = LoggerFactory.getLogger(UserProfileAnomalyKDEEvaluator.class);
-
-    @Override
-    public List<MLCallbackResult> detect(final String user, final String algorithm, UserActivityAggModel userActivity, UserProfileKDEModel aModel) {
-        List<MLCallbackResult> mlPredictionOutputList = new ArrayList<MLCallbackResult>();
-        RealMatrix inputData = userActivity.matrix();
-
-        double[] probabilityEstimation = new double[inputData.getRowDimension()];
-        for(int i=0; i < probabilityEstimation.length; i++)
-            probabilityEstimation[i] = 1.0;
-
-        boolean[][] anomalyFeature = new boolean[inputData.getRowDimension()][inputData.getColumnDimension()];
-
-        for(int i=0; i < anomalyFeature.length;i++){
-            for(int j=0; j < anomalyFeature[i].length; j++){
-                anomalyFeature[i][j] = false;
-            }
-        }
-
-        if(aModel == null){
-            LOG.info("No model available for this uer, returning");
-            return null;
-        }
-
-        Map<String,String> context = new HashMap<String,String>(){{
-            put(UserProfileConstants.USER_TAG,user);
-            put(UserProfileConstants.ALGORITHM_TAG,algorithm);
-        }};
-
-        for(int i=0; i<inputData.getRowDimension();i++){
-            List<String> cmds = JavaConversions.seqAsJavaList(userActivity.cmdTypes());
-            if(inputData.getColumnDimension() != cmds.size()){
-                LOG.error("Test data is not with same dimension as training, aborting...");
-                return null;
-            }else{
-
-                UserCommandStatistics[] listStats = aModel.statistics();
-
-                for(int j=0; j < inputData.getColumnDimension(); j++){
-//                    LOG.info("mean for j=" + j + " is:" + listStats[j].getMean());
-//                    LOG.info("stddev for j=" + j + " is:" + listStats[j].getStddev());
-                    if(listStats[j].isLowVariant()){
-//                        LOG.info(listStats[j].getCommandName() + " is low variant for user: " + user);
-                        if(inputData.getEntry(i, j) > listStats[j].getMean()){
-                            probabilityEstimation[i]*= Double.NEGATIVE_INFINITY;
-                            anomalyFeature[i][j] = true;
-                        }
-                    }else{
-                        double stddev = listStats[j].getStddev();
-                        //LOG.info("stddev: " + stddev);
-                        double mean = listStats[j].getMean();
-                        //LOG.info("mean: " + mean);
-                        double sqrt2PI = Math.sqrt(2.0*Math.PI);
-                        //LOG.info("sqrt2PI: " + sqrt2PI);
-                        double denominatorFirstPart = sqrt2PI*stddev;
-                        //LOG.info("denominatorFirstPart: " + denominatorFirstPart);
-                        double squareMeanNormal = Math.pow((inputData.getEntry(i, j) - mean), 2);
-                        //LOG.info("squareMeanNormal: " + squareMeanNormal);
-                        double twoPowStandardDev = Math.pow(stddev, 2);
-                        //LOG.info("twoPowStandardDev: " + twoPowStandardDev);
-                        double twoTimesTwoPowStandardDev = 2.0*twoPowStandardDev;
-                        //LOG.info("twoTimesTwoPowStandardDev: " + twoTimesTwoPowStandardDev);
-
-                        double tempVal = ((1.00/denominatorFirstPart)
-                                *(Math.exp(-(squareMeanNormal/twoTimesTwoPowStandardDev))));
-                        probabilityEstimation[i] *= tempVal;
-                        //LOG.info("probabilityEstimation: " + probabilityEstimation[i]);
-                        if((inputData.getEntry(i, j) - mean) > 2*stddev)
-                            anomalyFeature[i][j] = true;
-                    }
-                }
-
-            }
-        }
-
-        for(int i=0; i < probabilityEstimation.length;i++){
-            MLCallbackResult callBackResult = new MLCallbackResult();
-            callBackResult.setContext(context);
-            //LOG.info("probability estimation for data @" + i + " is: " + probabilityEstimation[i]);
-            if(probabilityEstimation[i] < aModel.maxProbabilityEstimate()){
-                callBackResult.setAnomaly(true);
-                for(int col = 0 ; col < anomalyFeature[i].length; col++){
-                    //LOG.info("feature anomaly? " + (featureVals[col] == true));
-                    if(anomalyFeature[i][col] == true){
-                        callBackResult.setFeature(aModel.statistics()[col].getCommandName());
-                    }
-                }
-            }else{
-                callBackResult.setAnomaly(false);
-            }
-
-            callBackResult.setTimestamp(userActivity.timestamp());
-            List<String> datapoints = new ArrayList<String>();
-            double[] rowVals = userActivity.matrix().getRow(i);
-            for(double rowVal:rowVals)
-                datapoints.add(rowVal+"");
-            callBackResult.setDatapoints(datapoints);
-            callBackResult.setId(user);
-            callBackResult.setAlgorithm(UserProfileConstants.KDE_ALGORITHM);
-            mlPredictionOutputList.add(callBackResult);
-        }
-        return mlPredictionOutputList;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider b/eagle-security/eagle-security-userprofile/detection/src/main/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
deleted file mode 100644
index d7a2754..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.eagle.ml.impl.MLPolicyEvaluatorServiceProviderImpl
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-batch.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-batch.conf b/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-batch.conf
deleted file mode 100644
index 8f2a690..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-batch.conf
+++ /dev/null
@@ -1,63 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{
-	"deployInstanceIdentifier" : {
-		"programId" : "mlProgramId"
-	},
-	"envContextConfig" : {
-		"env" : "storm",
-		"topologyName" : "userProfileDetectionTopology",
-		"mode" : "local",
-		"parallelismConfig" : {
-
-		},
-		"stormConfigFile":"userprofile.storm.yaml"
-	},
-	"dataSourceConfig": {
-		"topic" : "hdfs_audit_agg",
-		"zkConnection" : "localhost:2181",
-		"zkConnectionTimeoutMS" : 15000,
-		"consumerGroupId" : "EagleConsumer",
-		"fetchSize" : 1048586,
-		"deserializerClass" : "org.apache.eagle.security.userprofile.model.UserActivityAggModelDeserializer",
-		"transactionZKServers" : "localhost",
-		"transactionZKPort" : 2181,
-		"transactionZKRoot" : "/brokers/topics",
-		"transactionStateUpdateMS" : 2000
-	},
-	"eagleProps" : {
-		"site":"sandbox",
-		"application":"userProfile",
-		"eagleService": {
-			"host": "localhost",
-			"port": "38080",
-			"username": "admin",
-			"password": "secret"
-		}
-	},
-	"alertExecutorConfigs" : {
-		"userProfileAnomalyDetectionExecutor" : {
-			"parallelism" : 1,
-			"partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner",
-			"needValidation":true
-		}
-	},
-	"dynamicConfigSource" : {
-		"enabled"          : true,
-		"initDelayMillis"  : 0,
-		"delayMillis"      : 5000
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-stream.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-stream.conf b/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-stream.conf
deleted file mode 100644
index 09bec08..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/application-stream.conf
+++ /dev/null
@@ -1,65 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{
-	"deployInstanceIdentifier" : {
-		"programId" : "mlProgramId"
-	},
-	"envContextConfig" : {
-		"env" : "storm",
-		"topologyName" : "userProfileDetectionTopology",
-		"mode" : "local",
-		"parallelismConfig" : {
-
-		},
-		"stormConfigFile":"userprofile.storm.yaml"
-	},
-	"dataSourceConfig": {
-		"topic" : "sandbox_hdfs_audit_log",
-		"zkConnection" : "localhost:2181",
-		"zkConnectionTimeoutMS" : 15000,
-		"consumerGroupId" : "eagle.userprofile.detection",
-		"fetchSize" : 1048586,
-		"deserializerClass" : "org.apache.eagle.security.auditlog.HdfsAuditLogKafkaDeserializer",
-		"transactionZKServers" : "localhost",
-		"transactionZKPort" : 2181,
-		"transactionZKRoot" : "/brokers/topics",
-		"transactionStateUpdateMS" : 2000
-	},
-	"eagleProps" : {
-		"site":"sandbox",
-		"application":"userProfile",
-		"userProfileFeatures":"",
-		"userProfileMode":"streaming",
-		"eagleService": {
-			"host": "localhost",
-			"port": 38080,
-			"username": "admin",
-			"password": "secret"
-		}
-	},
-	"alertExecutorConfigs" : {
-		"userProfileAnomalyDetectionExecutor" : {
-			"parallelism" : 1,
-			"partitioner" : "org.apache.eagle.policy.DefaultPolicyPartitioner",
-			"needValidation":true
-		}
-	},
-	"dynamicConfigSource" : {
-		"enabled"          : true,
-		"initDelayMillis"  : 0,
-		"delayMillis"      : 30000
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/log4j.properties b/eagle-security/eagle-security-userprofile/detection/src/main/resources/log4j.properties
deleted file mode 100644
index 3499c46..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=INFO, stdout
-
- eagle.log.dir=./logs
- eagle.log.file=eagle.log
-
-# standard output
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
-
-# Daily Rolling File Appender
- log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
- log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file}
- log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-## 30-day backup
-# log4j.appender.DRFA.MaxBackupIndex=30
- log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/ml-policyDef-UserProfile.txt
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/ml-policyDef-UserProfile.txt b/eagle-security/eagle-security-userprofile/detection/src/main/resources/ml-policyDef-UserProfile.txt
deleted file mode 100644
index d217229..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/ml-policyDef-UserProfile.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-   "type":"MachineLearning",
-   "fileTypes":[
-      {
-         "type":"CSV",
-         "containsHeader":"true",
-         "fileParserClass":"eagle.ml.CSVFileFormatParser"
-      }
-   ],
-   "alertContext":{
-      "site":"sandbox",
-      "component":"testComponent",
-      "description":"ML based user profile anomaly detection",
-      "severity":"WARNING",
-      "notificationByEmail":"true"
-   },
-   "algorithms":[
-      {
-         "name":"eagle.security.userprofile.util.EigenBasedAnomalyDetection",
-         "description":"EigenBasedAnomalyDetection",
-         "modelPath":"/models/",
-         "featureSet":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck",
-         "type":"CSV"
-      },
-      {
-         "name":"eagle.security.userprofile.util.KDEBasedAnomalyDetection",
-         "description":"DensityBasedAnomalyDetection",
-         "modelPath":"/models/",
-         "featureSet":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck",
-         "type":"CSV"
-      }
-   ]
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/samplePolicy.json
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/samplePolicy.json b/eagle-security/eagle-security-userprofile/detection/src/main/resources/samplePolicy.json
deleted file mode 100644
index 4e63ae1..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/samplePolicy.json
+++ /dev/null
@@ -1,18 +0,0 @@
-[
-  {
-    "prefix": "alertdef",
-    "tags":{
-       "programId" : "hdfsAuditLogMonitoring",
-       "alertExecutorId" : "hdfsAuditLogAlertExecutor",
-       "policyId" : "insecureCopy",
-       "policyType" : "siddhiCEPEngine"
-   },
-     "desc": "insecure copy policy",
-     "policyDef": "{\"type\":\"siddhiCEPEngine\",\"expression\":\"from hdfsAuditLogEventStream[cmd=='open' and src=='/tmp/private']#window.time(2 sec) select cmd, dst, host, src, user, timestamp, allowed insert into outputStream ;\",\"alertContext\":{\"description\":\"insecure copy file\"}}",
-     "dedupeDef" : "{\"alertDedupIntervalMin\":\"1\",\"emailDedupIntervalMin\":\"1\"}",
-     "notificationDef" : "",
-     "remediationDef" : "",
-     "enabled" : "true"
-   }
-]
-

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/main/resources/userprofile.storm.yaml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/main/resources/userprofile.storm.yaml b/eagle-security/eagle-security-userprofile/detection/src/main/resources/userprofile.storm.yaml
deleted file mode 100644
index 274aa2c..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/main/resources/userprofile.storm.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-# 
-#    http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-### topology.* configs are for specific executing storms
-topology.enable.message.timeouts: true
-topology.debug: false
-topology.workers: 1
-topology.acker.executors: 10
-topology.tasks: null
-# maximum amount of time a message has to complete before it's considered failed
-topology.message.timeout.secs: 60000
-topology.receiver.buffer.size: 8
-topology.transfer.buffer.size: 1024
-topology.executor.receive.buffer.size: 16384
-topology.executor.send.buffer.size: 16384
-storm.zookeeper.session.timeout: 600000
-storm.zookeeper.connection.timeout: 90000

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserActivityAggregator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserActivityAggregator.java b/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserActivityAggregator.java
deleted file mode 100644
index 30bfaa3..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserActivityAggregator.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-package org.apache.eagle.security.userprofile;
-
-import org.apache.eagle.common.DateTimeUtil;
-import org.apache.eagle.datastream.Collector;
-import org.apache.eagle.security.userprofile.impl.UserActivityAggregatorImpl;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModelEntity;
-import org.joda.time.Period;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import scala.Tuple2;
-
-import java.text.ParseException;
-import java.util.*;
-
-@Ignore
-public class TestUserActivityAggregator {
-    private UserActivityAggregator aggregator_0;
-    private UserActivityAggregator aggregator_1;
-
-    @Before
-    public void setUp(){
-        aggregator_0 = new UserActivityAggregatorImpl(Arrays.asList(UserProfileConstants.DEFAULT_CMD_TYPES),Period.parse("PT1m"),"test",0);
-        aggregator_1 = new UserActivityAggregatorImpl(Arrays.asList(UserProfileConstants.DEFAULT_CMD_TYPES),Period.parse("PT1m"), "test", 5000);
-    }
-
-    @Test
-    public void testAggWithoutSafeWindow() throws ParseException {
-        SimpleCollector collector = new SimpleCollector();
-        // yyyy-MM-dd HH:mm:ss,SSS
-        aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:00,000"),"user1","open"),collector);
-        aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:01,000"),"user1","open"),collector);
-        aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:02,000"),"user1","open"),collector);
-
-        aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:59,000"),"user2","open"),collector);
-
-        aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:01,000"),"user1","open"),collector);
-        aggregator_0.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:02,000"),"user2","open"),collector);
-
-        Assert.assertEquals(2, collector.getResult().size());
-        Assert.assertEquals("user2",collector.getResult().get(0)._1());
-        Assert.assertEquals("user1",collector.getResult().get(1)._1());
-    }
-
-    @Test
-    public void testAggWithSafeWindow() throws ParseException {
-        SimpleCollector collector = new SimpleCollector();
-        aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:00,000"),"user1","open"),collector);  // create window
-        aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:01,000"),"user1","rename"),collector);
-        aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:59,000"),"user1","delete"),collector);
-
-        aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:01,000"),"user1","open"),collector);
-        aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:01:58,000"),"user2","open"),collector); // inside safe-window
-
-        aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:02:06,000"),"user2","open"),collector); //
-
-        aggregator_1.accumulate(event(DateTimeUtil.humanDateToMilliseconds("2015-09-29 01:06:02,000"),"user2","open"),collector); //  outside safe-window
-
-        Assert.assertEquals(4, collector.getResult().size());
-        Assert.assertEquals("user2",collector.getResult().get(0)._1());
-        Assert.assertEquals("user1",collector.getResult().get(1)._1());
-    }
-
-
-    private Map<String,Object> event(final long timestamp, final String user, final String cmd){
-        return new HashMap<String,Object>(){{
-            put("timestamp",timestamp);
-            put("user",user);
-            put("cmd",cmd);
-        }};
-    }
-
-    private class SimpleCollector implements Collector<Tuple2<String, UserActivityAggModelEntity>>{
-        private List<Tuple2<String, UserActivityAggModelEntity>> result = new ArrayList<>();
-        @Override
-        public void collect(Tuple2<String, UserActivityAggModelEntity> tuple) {
-            result.add(tuple);
-        }
-
-        public List<Tuple2<String, UserActivityAggModelEntity>> getResult(){
-            return result;
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserProfileAnomalyEigenEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserProfileAnomalyEigenEvaluator.java b/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserProfileAnomalyEigenEvaluator.java
deleted file mode 100644
index eddea94..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/test/java/org/apache/eagle/security/userprofile/TestUserProfileAnomalyEigenEvaluator.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- * 
- */
-package org.apache.eagle.security.userprofile;
-
-import org.apache.eagle.ml.model.MLCallbackResult;
-import org.apache.eagle.security.userprofile.impl.UserProfileAnomalyEigenEvaluator;
-import org.apache.eagle.security.userprofile.model.UserActivityAggModel;
-import org.apache.eagle.security.userprofile.model.UserCommandStatistics;
-import org.apache.eagle.security.userprofile.model.UserProfileEigenModel;
-import org.apache.commons.math3.linear.Array2DRowRealMatrix;
-import org.apache.commons.math3.linear.ArrayRealVector;
-import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.commons.math3.linear.RealVector;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.collection.JavaConversions;
-import scala.collection.Seq;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class TestUserProfileAnomalyEigenEvaluator {
-
-    private static Logger LOG = LoggerFactory.getLogger(TestUserProfileAnomalyEigenEvaluator.class);
-
-    @Test
-    public void testDetect(){
-        UserProfileAnomalyEigenEvaluator eigenEvaluator = new UserProfileAnomalyEigenEvaluator();
-        String[] testCmdType = {"getfileinfo", "open", "listStatus", "setTimes", "setPermission", "rename",
-                                    "mkdirs", "create", "setReplication", "contentSummary", "delete", "setOwner",
-                                    "fsck"};
-        List<String> tmpCmdTypesAsList = new ArrayList<String>();
-        tmpCmdTypesAsList = Arrays.asList(testCmdType);
-        Seq<String> testCmd = JavaConversions.asScalaBuffer(tmpCmdTypesAsList);
-        String testSite = "sandbox";
-        long testTimestamp = 14054440;
-        String testUser = "test_user";
-        RealMatrix testMatrix = new Array2DRowRealMatrix(1,testCmdType.length);
-        for(int i=0; i < testMatrix.getColumnDimension();i++)
-            testMatrix.addToEntry(0, i, 3.0);
-
-        UserActivityAggModel testAggModel = new UserActivityAggModel(testUser, testMatrix, testCmd,testSite, testTimestamp);
-
-        Long testVersion = new Long(1);
-        RealMatrix testUMat = new Array2DRowRealMatrix(testCmdType.length,testCmdType.length);
-        RealMatrix testDiagonalMat = new Array2DRowRealMatrix(testCmdType.length, testCmdType.length);
-
-        for(int i=0; i< testCmdType.length; i++){
-            for(int  j=0; j < testCmdType.length; j++){
-                testUMat.addToEntry(i,j,1.0);
-                testDiagonalMat.addToEntry(i,j,1.0);
-            }
-        }
-
-        int dimension = testCmdType.length -1;
-        double[] minVector = new double[testCmdType.length];
-        double[] maxVector = new double[testCmdType.length];
-        for(int i=0; i < minVector.length;i++) {
-            minVector[i] = 1;
-            maxVector[i] = 1;
-        }
-
-        RealVector testMinVec = new ArrayRealVector(minVector);
-        RealVector testMaxVec = new ArrayRealVector(maxVector);
-        RealVector[] testPCs = new ArrayRealVector[testCmdType.length];
-
-        for(int i =0; i < testCmdType.length; i++) {
-            testPCs[i] = new ArrayRealVector(testCmdType.length);
-            for(int j = 0; j < testPCs[i].getDimension(); j++){
-                testPCs[i].addToEntry(j, 1.0);
-            }
-        }
-
-        RealVector testMaxL2Norm = new ArrayRealVector(maxVector);
-        RealVector testMinL2Norm = new ArrayRealVector(minVector);
-        UserCommandStatistics userCommandStatistics[] = new UserCommandStatistics[testCmdType.length];
-
-        for(int i=0; i< testCmdType.length;i++){
-            userCommandStatistics[i] = new UserCommandStatistics();
-            userCommandStatistics[i].setCommandName(testCmdType[i]);
-            userCommandStatistics[i].setLowVariant(false);
-            userCommandStatistics[i].setMean(1.0);
-            userCommandStatistics[i].setStddev(1.0);
-        }
-
-        UserProfileEigenModel testEigenModel = new UserProfileEigenModel(testVersion,testSite,testUser,testUMat,testDiagonalMat,dimension, testMinVec, testMaxVec, testPCs, testMaxL2Norm, testMinL2Norm, userCommandStatistics);
-
-        List<MLCallbackResult> testResults = eigenEvaluator.detect("test_user", "eigen", testAggModel, testEigenModel);
-
-        Assert.assertEquals(testResults.size(), testMatrix.getRowDimension());
-        for(MLCallbackResult result: testResults){
-            Assert.assertEquals(result.isAnomaly(), true);
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/test/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/test/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider b/eagle-security/eagle-security-userprofile/detection/src/test/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
deleted file mode 100644
index e363e67..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/test/resources/META-INF/services/org.apache.eagle.policy.PolicyEvaluatorServiceProvider
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# org.apache.eagle.ml.legacy.MachineLearningPolicyEvaluatorServiceProviderImpl
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/test/resources/ml-policyDef-UserProfile.txt
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/test/resources/ml-policyDef-UserProfile.txt b/eagle-security/eagle-security-userprofile/detection/src/test/resources/ml-policyDef-UserProfile.txt
deleted file mode 100644
index bb38d4a..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/test/resources/ml-policyDef-UserProfile.txt
+++ /dev/null
@@ -1 +0,0 @@
-{"type":"MachineLearning","fileTypes":[{"type":"CSV","containsHeader":"true","fileParserClass":"eagle.ml.CSVFileFormatParser"}],"alertContext":{"cluster":"testCluster","datacenter":"testDataCenter","component":"testComponent","description":"ML based user profile anomaly detection","severity":"WARNING","notificationByEmail":"true"},"algorithms":[{"name":"eagle.security.userprofile.legacy.legacy.util.EigenBasedAnomalyDetection","description":"EigenBasedAnomalyDetection","modelPath":"/models/","featureSet":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck","type":"CSV"},{"name":"eagle.security.userprofile.legacy.util.KDEBasedAnomalyDetection","description":"DensityBasedAnomalyDetection","modelPath":"/models/","featureSet":"getfileinfo, open, listStatus, setTimes, setPermission, rename, mkdirs, create, setReplication, contentSummary, delete, setOwner, fsck","type":"CSV"}]}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/detection/src/test/resources/storm.yaml.1
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/detection/src/test/resources/storm.yaml.1 b/eagle-security/eagle-security-userprofile/detection/src/test/resources/storm.yaml.1
deleted file mode 100644
index 2ea7262..0000000
--- a/eagle-security/eagle-security-userprofile/detection/src/test/resources/storm.yaml.1
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### topology.* configs are for specific executing storms
-topology.enable.message.timeouts: true
-topology.debug: false
-topology.workers: 1
-topology.acker.executors: 10
-topology.tasks: null
-# maximum amount of time a message has to complete before it's considered failed
-topology.message.timeout.secs: 60000
-topology.receiver.buffer.size: 8
-topology.transfer.buffer.size: 1024
-topology.executor.receive.buffer.size: 16384
-topology.executor.send.buffer.size: 16384
-storm.zookeeper.session.timeout: 600000
-storm.zookeeper.connection.timeout: 90000

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/pom.xml b/eagle-security/eagle-security-userprofile/pom.xml
deleted file mode 100644
index 0c112d3..0000000
--- a/eagle-security/eagle-security-userprofile/pom.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~    http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>eagle-security-parent</artifactId>
-        <groupId>org.apache.eagle</groupId>
-        <version>0.5.0-incubating-SNAPSHOT</version>
-        <relativePath>../pom.xml</relativePath>
-    </parent>
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>eagle-security-userprofile-parent</artifactId>
-    <packaging>pom</packaging>
-
-    <modules>
-        <module>common</module>
-        <module>detection</module>
-        <module>training</module>
-    </modules>
-</project>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/README.md
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/README.md b/eagle-security/eagle-security-userprofile/training/README.md
deleted file mode 100644
index 79a42e4..0000000
--- a/eagle-security/eagle-security-userprofile/training/README.md
+++ /dev/null
@@ -1,54 +0,0 @@
-User Profiling Offline Training 
-===============================
-
-Quick Start
------------
-1. Compile and build package
-	
-		mvn clean compile package -DskipTests
-	
-2. Run with following command or directly on IDE
-	
-		spark-submit --class eagle.security.auditlog.AuditlogTrainingMain --master local target/eagle-security-mltraining-spark-0.1.0.jar
-
-Problems
---------
-1. **Problem**: 2015-07-19 17:18:42,083 ERROR [main] spark.SparkContext (Logging.scala:logError(96)) - Error initializing SparkContext. java.net.BindException: Failed to bind to: /10.249.64.134:0: Service 'sparkDriver' failed after 16 retries!
-   **Solution**: Add environment variables
-
-		export SPARK_LOCAL_IP=127.0.0.1
-		export SPARK_MASTER_IP=127.0.0.1
-
-2. **Problem**: Detected both slf4j-log4j12 and log4j-over-slf4j in classpath
-
-   **Solution**: Exclude log4 and slf4j related dependencies
-
-		<exclusions>
-		<exclusion>
-		   <groupId>org.slf4j</groupId>
-		   <artifactId>slf4j-log4j12</artifactId>
-		</exclusion>
-		<exclusion>
-		   <groupId>org.slf4j</groupId>
-		   <artifactId>log4j-over-slf4j</artifactId>
-		</exclusion>
-		</exclusions>
-
-3. **Problem**: fasterxml related class method not found
-
-   **Solution**: Exclude following modules in version `2.4.1`:
-   	
-		"com.fasterxml.jackson.core:jackson-core"
-		"com.fasterxml.jackson.core:jackson-databind"
-		"com.fasterxml.jackson.module:jackson-module-scala_2.10" 
-		
-   but explicitly use `2.3.1` instead
-
-4. **Problem**: “org.apache.commons.math3.exception.MathIllegalArgumentException: insufficient data: only 10 rows and 1 columns.” breaks spark on distributed environment, but works locally.
-
-   **Solution**: It's because Spark natively depends on math3 version "3.1.1".
-
-5. **Problem**: Exception in thread "delete Spark temp dirs" java.lang.NoClassDefFoundError: Could not initialize class org.apache.log4j.LogManager
-
-   **Solution**: TODO
-

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-scheduler.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-scheduler.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-scheduler.sh
deleted file mode 100755
index 1b7907b..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-scheduler.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-which spark-submit 1>/dev/null 2>&1
-
-if [ $? != 0 ];then
-	echo "Error: spark-submit not found, please add spark-summit into environment PATH, exit code: 1"
-	exit 1
-fi
-
-cd $(dirname $0)/../
-
-export JAR_FILE=$(ls `pwd`/target/eagle-security-userprofile-training-*-assembly.jar)
-export DRIVER_CLASSPATH=$(ls `pwd`/target/lib/asm-3.1.jar):$(ls `pwd`/target/lib/commons-math3-3.5.jar)
-
-#if [ ! -e $JAR_FILE ]; then
-#	echo "$JAR_FILE not exists, build package ..."
-#	mvn clean package -DskipTests
-#	JAR_FILE=$(ls `pwd`/target/eagle-security-userprofile-training-*-assembly.jar)
-#else
-#	echo "$JAR_FILE already exists"
-#
-#fi
-
-# Force clean and re-package
-mvn clean package -DskipTests
-
-export AUDIT_LOG_PATH=$(ls `pwd`/src/main/resources/hdfs-audit.log)
-
-mvn exec:java -DskipTests -Dexec.mainClass="eagle.security.userprofile.daemon.Scheduler"  \
-	-Dexec.args="-D eagle.userprofile.driver-classpath=$DRIVER_CLASSPATH -D eagle.userprofile.jar=$JAR_FILE -D eagle.userprofile.training-audit-path=$AUDIT_LOG_PATH -D eagle.userprofile.detection-audit-path=$AUDIT_LOG_PATH $@"
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-training.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-training.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-training.sh
deleted file mode 100755
index f6da96a..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-training.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-export SPARK_LOCAL_IP=127.0.0.1
-export SPARK_MASTER_IP=127.0.0.1
-
-cd $(dirname $0)/../
-
-mvn exec:java -DskipTests -Dexec.mainClass="eagle.security.userprofile.UserProfileTrainingCLI" -Dexec.args="$@"
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-kafka.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-kafka.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-kafka.sh
deleted file mode 100755
index c4e53e1..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-kafka.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-export SPARK_LOCAL_IP=127.0.0.1
-export SPARK_MASTER_IP=127.0.0.1
-
-cd $(dirname $0)/../
-
-export HDFS_AUDIT=`pwd`/src/main/resources/hdfs-audit.log
-export OUTPUT_PATH=/tmp/eagle-userprofile-output
-
-rm -rf $OUTPUT_PATH 1>/dev/null 2>&1
-
-mvn exec:java -DskipTests -Dexec.mainClass="eagle.security.userprofile.UserProfileTrainingCLI" -Dexec.args="--site sandbox --input $HDFS_AUDIT --kafka-props topic=hdfs_audit_agg,metadata.broker.list=sandbox.hortonworks.com:6667"
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-localfile.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-localfile.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-localfile.sh
deleted file mode 100755
index bd166d6..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-localfile.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-export SPARK_LOCAL_IP=127.0.0.1
-export SPARK_MASTER_IP=127.0.0.1
-
-cd $(dirname $0)/../
-
-export HDFS_AUDIT=`pwd`/src/main/resources/hdfs-audit.log
-export OUTPUT_PATH=/tmp/eagle-userprofile-output
-
-rm -rf $OUTPUT_PATH 1>/dev/null 2>&1
-
-echo ""
-echo "CMD: mvn exec:java -DskipTests -Dexec.mainClass=\"eagle.security.userprofile.UserProfileTrainingCLI\" -Dexec.args=\"--input $HDFS_AUDIT\""
-echo ""
-mvn exec:java -DskipTests -Dexec.mainClass="eagle.security.userprofile.UserProfileTrainingCLI" -Dexec.args="--site sandbox --input $HDFS_AUDIT --output $OUTPUT_PATH"
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-service.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-service.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-service.sh
deleted file mode 100755
index c35f3d1..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-service.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-export SPARK_LOCAL_IP=127.0.0.1
-export SPARK_MASTER_IP=127.0.0.1
-
-cd $(dirname $0)/../
-
-export HDFS_AUDIT=`pwd`/src/main/resources/hdfs-audit.log
-
-rm -rf $OUTPUT_PATH 1>/dev/null 2>&1
-
-echo ""
-echo "Execute: mvn exec:java -DskipTests -Dexec.mainClass=\"eagle.security.userprofile.UserProfileTrainingCLI\" -Dexec.args=\"--input $HDFS_AUDIT\""
-echo ""
-mvn exec:java -DskipTests -Dexec.mainClass="eagle.security.userprofile.UserProfileTrainingCLI" -Dexec.args="--site sandbox --input $HDFS_AUDIT --service-host localhost --service-port 9098 --service-username admin --service-password secret"
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-spark.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-spark.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-spark.sh
deleted file mode 100755
index e263928..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/run-with-spark.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-which spark-submit 1>/dev/null 2>&1
-
-if [ $? != 0 ];then
-	echo "Error: spark-submit not found, please add spark-summit into environment PATH, exit code: 1"
-	exit 1
-fi
-
-cd $(dirname $0)/../
-
-cmd="mvn clean compile package -DskipTests"
-echo -e "Execute: $cmd\n"
-$cmd
-
-if [ $? != 0 ];then
-	echo "-----"
-	echo "Failed to compile and package project"
-	exit 1
-else
-	echo "Success to compile and package project"
-fi
-
-rm -rf /tmp/eagle-userprofile-output 1>/dev/null 2>&1
-cmd="spark-submit --class eagle.security.userprofile.UserProfileTrainingCLI `ls target/eagle-security-userprofile-training-*-assembly.jar` --master local[10] --site sandbox --input `pwd`/src/main/resources/hdfs-audit.log --output /tmp/eagle-userprofile-output"
-echo -e "Execute: $cmd\n"
-$cmd
-
-exit_code=$?
-if [ $exit_code != 0 ];then
-	echo "-----"
-	echo "Failed to execute job on spark, exit code: $exit_code"
-	exit 1
-else
-	echo "-----"
-	echo "Success to execute job on spark, output: /tmp/eagle-userprofile-output, exit code: $exit_code"
-fi
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/dev-supports/trigger-onetime-training.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/dev-supports/trigger-onetime-training.sh b/eagle-security/eagle-security-userprofile/training/dev-supports/trigger-onetime-training.sh
deleted file mode 100755
index 3907a96..0000000
--- a/eagle-security/eagle-security-userprofile/training/dev-supports/trigger-onetime-training.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-export SERVICE_HOST="localhost"
-export SERVICE_PORT=9098
-
-curl -H "Content-Type:application/json" -H "X-HTTP-Method-Override: POST" -XPOST "${SERVICE_HOST}:${SERVICE_PORT}/eagle-service/rest/entities?serviceName=ScheduleTaskService" -d "[{\"timestamp\":1441554892716,\"tags\": {\"site\":\"sandbox\",\"type\":\"USER_PROFILE_TRAINING\"},\"status\":\"INITIALIZED\",\"detail\":\"Newly created command\"}]" 2>/dev/null
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/8b646715/eagle-security/eagle-security-userprofile/training/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-userprofile/training/pom.xml b/eagle-security/eagle-security-userprofile/training/pom.xml
deleted file mode 100644
index 430cf95..0000000
--- a/eagle-security/eagle-security-userprofile/training/pom.xml
+++ /dev/null
@@ -1,219 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~    http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>eagle-security-userprofile-parent</artifactId>
-        <groupId>org.apache.eagle</groupId>
-        <version>0.5.0-incubating-SNAPSHOT</version>
-        <relativePath>../pom.xml</relativePath>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>eagle-security-userprofile-training</artifactId>
-
-    <dependencies>
-        <!-- spark needs asm 3, so we should use extcos 0.3b -->
-        <dependency>
-            <groupId>net.sf.extcos</groupId>
-            <artifactId>extcos</artifactId>
-            <version>${extcos3.version}</version>
-        </dependency>
-
-        <!-- Eagle -->
-        <dependency>
-            <groupId>org.apache.eagle</groupId>
-            <artifactId>eagle-security-common</artifactId>
-            <version>${project.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>javax.servlet</groupId>
-                    <artifactId>servlet-api</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.storm</groupId>
-                    <artifactId>storm-core</artifactId>
-                </exclusion>
-                <exclusion>
-                    <artifactId>servlet-api</artifactId>
-                    <groupId>org.mortbay.jetty</groupId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>joda-time</groupId>
-            <artifactId>joda-time</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.joda</groupId>
-            <artifactId>joda-convert</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.eagle</groupId>
-            <artifactId>eagle-security-userprofile-common</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-
-        <!-- unit test -->
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-math3</artifactId>
-        </dependency>
-
-        <!-- spark -->
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-core_${scala.version}</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.github.scopt</groupId>
-            <artifactId>scopt_${scala.version}</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_${scala.version}</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.scalatest</groupId>
-            <artifactId>scalatest_${scala.version}</artifactId>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-    <build>
-        <resources>
-            <resource>
-                <directory>src/main/resources</directory>
-            </resource>
-        </resources>
-        <plugins>
-            <plugin>
-                <groupId>org.scala-tools</groupId>
-                <artifactId>maven-scala-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>scala-compile-first</id>
-                        <phase>process-resources</phase>
-                        <goals>
-                            <goal>add-source</goal>
-                            <goal>compile</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>scala-test-compile</id>
-                        <phase>process-test-resources</phase>
-                        <goals>
-                            <goal>testCompile</goal>
-                        </goals>
-                    </execution>
-
-                </executions>
-            </plugin>
-
-            <plugin>
-                <groupId>org.scalatest</groupId>
-                <artifactId>scalatest-maven-plugin</artifactId>
-                <configuration>
-                    <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
-                    <junitxml>.</junitxml>
-                    <filereports>TestSuite.txt</filereports>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>test</id>
-                        <goals>
-                            <goal>test</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <phase>compile</phase>
-                        <goals>
-                            <goal>compile</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-dependency-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>copy-dependencies</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <configuration>
-                <useFile>false</useFile>
-                <disableXmlReport>true</disableXmlReport>
-                <!-- If you have classpath issue like NoDefClassError,... -->
-                <!-- useManifestOnlyJar>false</useManifestOnlyJar -->
-                <includes>
-                <include>**/*Test.*</include>
-                <include>**/Test*.*</include>
-                <include>**/*Suite.*</include>
-                </includes>
-                </configuration>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <configuration>
-                    <descriptor>src/assembly/assembly.xml</descriptor>
-                    <archive>
-                        <manifest>
-                            <mainClass>org.apache.eagle.security.userprofile.UserProfileTrainingCLI</mainClass>
-                        </manifest>
-                    </archive>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                        <configuration>
-                            <tarLongFileMode>posix</tarLongFileMode>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>


Mime
View raw message