metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [1/3] incubator-metron git commit: METRON-690: Create a DSL-based timestamp lookup for profiler to enable sparse windows closes apache/incubator-metron#450
Date Fri, 24 Feb 2017 20:53:24 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master 7abd7e8a2 -> 84d347195


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/84d34719/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/IntervalPredicateTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/IntervalPredicateTest.java
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/IntervalPredicateTest.java
new file mode 100644
index 0000000..8a0ea62
--- /dev/null
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/IntervalPredicateTest.java
@@ -0,0 +1,89 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.client.stellar;
+
+import org.apache.commons.lang3.Range;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+public class IntervalPredicateTest {
+  @Test
+  public void testBasicTest() {
+    List<Range<Long>> intervals = new ArrayList<Range<Long>>() {{
+      add(Range.between(0L, 10L));
+      add(Range.between(20L, 30L));
+      add(Range.between(40L, 50L));
+    }};
+    IntervalPredicate<Long> predicate = new IntervalPredicate.Identity(intervals);
+    Assert.assertTrue(predicate.test(0L));
+    Assert.assertTrue(predicate.test(10L));
+    Assert.assertTrue(predicate.test(5L));
+    Assert.assertFalse(predicate.test(51L));
+    Assert.assertFalse(predicate.test(15L));
+  }
+
+  @Test
+  public void testWithOverlap() {
+    List<Range<Long>> intervals = new ArrayList<Range<Long>>() {{
+      add(Range.between(0L, 10L));
+      add(Range.between(5L, 30L));
+      add(Range.between(40L, 50L));
+    }};
+    IntervalPredicate<Long> predicate = new IntervalPredicate.Identity(intervals);
+    Assert.assertTrue(predicate.test(0L));
+    Assert.assertTrue(predicate.test(5L));
+    Assert.assertTrue(predicate.test(30L));
+    Assert.assertTrue(predicate.test(10L));
+    Assert.assertFalse(predicate.test(51L));
+    Assert.assertTrue(predicate.test(15L));
+    Assert.assertFalse(predicate.test(31L));
+    Assert.assertTrue(predicate.test(45L));
+  }
+
+  @Test
+  public void testTrivialCase() {
+    List<Range<Long>> intervals = new ArrayList<Range<Long>>() {{
+      add(Range.between(0L, 10L));
+    }};
+    IntervalPredicate<Long> predicate = new IntervalPredicate.Identity(intervals);
+    Assert.assertTrue(predicate.test(0L));
+    Assert.assertTrue(predicate.test(5L));
+    Assert.assertTrue(predicate.test(10L));
+    Assert.assertFalse(predicate.test(51L));
+    Assert.assertFalse(predicate.test(15L));
+  }
+
+  @Test
+  public void testDegenerateCase() {
+    List<Range<Long>> intervals = new ArrayList<Range<Long>>() {{
+      add(Range.between(10L, 10L));
+    }};
+    IntervalPredicate<Long> predicate = new IntervalPredicate.Identity(intervals);
+    Assert.assertFalse(predicate.test(0L));
+    Assert.assertFalse(predicate.test(5L));
+    Assert.assertTrue(predicate.test(10L));
+    Assert.assertFalse(predicate.test(11L));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/84d34719/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
new file mode 100644
index 0000000..aafbe5c
--- /dev/null
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
@@ -0,0 +1,207 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.client.stellar;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.lang3.Range;
+import org.apache.metron.common.dsl.Context;
+import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
+import org.apache.metron.common.dsl.functions.resolver.SimpleFunctionResolver;
+import org.apache.metron.common.stellar.StellarProcessor;
+import org.apache.metron.profiler.ProfilePeriod;
+import org.apache.metron.profiler.client.window.WindowProcessor;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+public class WindowLookbackTest {
+
+  static FunctionResolver resolver;
+  static Context context;
+  @BeforeClass
+  public static void setup() {
+    resolver = new SimpleFunctionResolver()
+                    .withClass(GetProfile.class)
+                    .withClass(FixedLookback.class)
+                    .withClass(WindowLookback.class);
+    context = new Context.Builder()
+                    .with(Context.Capabilities.GLOBAL_CONFIG, () -> new HashMap<>())
+                    .build();
+  }
+
+  @Test
+  public void testSpecifyingConfig() throws Exception {
+    //we should be able to specify the config and have it take hold.  If we change the
+    //profile duration to 1 minute instead of 15 minutes (the default), then we should see
+    //the correct number of profiles.
+    long durationMs = 60000;
+    State state = test("1 hour", new Date()
+                      , Optional.of(
+                              ImmutableMap.of( ProfilerConfig.PROFILER_PERIOD.getKey(), 1
)
+                                   )
+                      ,Assertions.NOT_EMPTY,Assertions.CONTIGUOUS);
+    Assert.assertEquals(TimeUnit.HOURS.toMillis(1) / durationMs, state.periods.size());
+  }
+
+  @Test
+  public void testSpecifyingOnlySelector() {
+    String stellarStatement = "PROFILE_WINDOW('1 hour')";
+    Map<String, Object> variables = new HashMap<>();
+    StellarProcessor stellar = new StellarProcessor();
+    List<ProfilePeriod> periods = (List<ProfilePeriod>)stellar.parse( stellarStatement
+                                                                    , k -> variables.get(k)
+                                                                    , resolver
+                                                                    , context
+                                                                    );
+    Assert.assertEquals(TimeUnit.HOURS.toMillis(1) / getDurationMs(), periods.size());
+  }
+
+  @Test
+  public void testDenseLookback() throws Exception {
+    State state = test("1 hour", Assertions.NOT_EMPTY, Assertions.CONTIGUOUS);
+    Assert.assertEquals(TimeUnit.HOURS.toMillis(1) / getDurationMs(), state.periods.size());
+  }
+
+  @Test
+  public void testShiftedDenseLookback() throws Exception {
+    State state = test("from 2 hours ago to 30 minutes ago", Assertions.NOT_EMPTY
+                                                           , Assertions.CONTIGUOUS
+                                                           , Assertions.INTERVALS_CONTAIN_ALL_PERIODS
+                                                           );
+    Assert.assertEquals(TimeUnit.MINUTES.toMillis(90) / getDurationMs(), state.periods.size());
+  }
+
+  @Test
+  public void testShiftedSparseLookback() throws Exception {
+    State state = test("30 minute window every 1 hour from 2 hours ago to 30 minutes ago",
Assertions.NOT_EMPTY
+                                                                                        
, Assertions.DISCONTIGUOUS
+                                                                                        
, Assertions.INTERVALS_CONTAIN_ALL_PERIODS
+                                                                                        
);
+    Assert.assertEquals(TimeUnit.MINUTES.toMillis(60) / getDurationMs(), state.periods.size());
+  }
+
+  @Test
+  public void testEmptyDueToExclusions() throws Exception {
+    test("30 minute window every 24 hours from 7 days ago including saturdays excluding weekends",
Assertions.EMPTY);
+  }
+
+  @Test(expected= ParseException.class)
+  public void testErrorInSelector() throws Exception {
+    test("30 minute idow every 24 hours from 7 days ago including saturdays excluding weekends",
Assertions.EMPTY);
+  }
+
+  long getDurationMs() {
+    int duration = ProfilerConfig.PROFILER_PERIOD.getDefault(Integer.class);
+    TimeUnit unit = TimeUnit.valueOf(ProfilerConfig.PROFILER_PERIOD_UNITS.getDefault(String.class));
+    return unit.toMillis(duration);
+  }
+
+  public State test(String windowSelector, Assertions... assertions) {
+    return test(windowSelector, new Date(), Optional.empty(), assertions);
+  }
+
+  public State test(String windowSelector, Date now, Optional<Map<String, Object>>
config, Assertions... assertions) {
+
+    List<Range<Long>> windowIntervals = WindowProcessor.process(windowSelector).toIntervals(now.getTime());
+    String stellarStatement = "PROFILE_WINDOW('" + windowSelector + "', now"
+                            + (config.isPresent()?", config":"")
+                            + ")";
+    Map<String, Object> variables = new HashMap<>();
+    variables.put("now", now.getTime());
+    if(config.isPresent()) {
+      variables.put("config", config.get());
+    }
+    StellarProcessor stellar = new StellarProcessor();
+    List<ProfilePeriod> periods = (List<ProfilePeriod>)stellar.parse( stellarStatement
+                                                                    , k -> variables.get(k)
+                                                                    , resolver
+                                                                    , context
+                                                                    );
+    State state = new State(windowIntervals, periods);
+    for(Assertions assertion : assertions) {
+      Assert.assertTrue(assertion.name(), assertion.test(state));
+    }
+    return state;
+  }
+
+  private enum Assertions implements Predicate<State>{
+    EMPTY( state -> state.windowIntervals.isEmpty() && state.periods.isEmpty()
),
+    NOT_EMPTY( state -> !state.windowIntervals.isEmpty() && !state.periods.isEmpty()),
+    CONTIGUOUS( state -> {
+      if(state.periods.size() < 2) {
+        return true;
+      }
+      long duration = state.periods.get(1).getStartTimeMillis() - state.periods.get(0).getStartTimeMillis();
+      for(int i = 1;i < state.periods.size();++i) {
+        long left = state.periods.get(i - 1).getStartTimeMillis();
+        long right = state.periods.get(i).getStartTimeMillis();
+        if(right - left != duration) {
+          return false;
+        }
+      }
+      return true;
+    }),
+    DISCONTIGUOUS( state -> !Assertions.CONTIGUOUS.test(state)),
+    INTERVALS_CONTAIN_ALL_PERIODS( state -> {
+      List<Range<Long>> windowIntervals = state.windowIntervals;
+      List<ProfilePeriod> periods = state.periods;
+
+      Set<Range<Long>> foundIntervals = new HashSet<>();
+      for(ProfilePeriod period : periods) {
+        boolean found = false;
+        for(Range<Long> interval : windowIntervals) {
+          if(interval.contains(period.getStartTimeMillis())) {
+            foundIntervals.add(interval);
+            found = true;
+          }
+        }
+        if(!found) {
+          return false;
+        }
+      }
+      return foundIntervals.size() == windowIntervals.size();
+    })
+    ;
+    Predicate<State> predicate;
+    Assertions(Predicate<State> predicate) {
+      this.predicate = predicate;
+    }
+
+    @Override
+    public boolean test(State s) {
+      return predicate.test(s);
+    }
+  }
+
+  private static class State {
+    List<Range<Long>> windowIntervals;
+    List<ProfilePeriod> periods;
+    public State(List<Range<Long>> windowIntervals, List<ProfilePeriod>
periods) {
+      this.periods = periods;
+      this.windowIntervals = windowIntervals;
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/84d34719/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java
new file mode 100644
index 0000000..94f9e04
--- /dev/null
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java
@@ -0,0 +1,314 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.client.window;
+
+import org.apache.commons.lang3.Range;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class WindowProcessorTest {
+
+  @Test
+  public void testBaseCase() {
+    for (String text : new String[] {
+            "1 hour"
+            ,"1 hour(s)"
+            ,"1 hours"
+    }) {
+      Window w = WindowProcessor.process(text);
+      Date now = new Date();
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(1, intervals.size());
+      Assert.assertEquals(now.getTime(), (long)intervals.get(0).getMaximum());
+      Assert.assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(1), (long)intervals.get(0).getMinimum());
+    }
+  }
+
+  @Test
+  public void testDenseWindow() {
+    for (String text : new String[] {
+            "from 2 hours ago to 30 minutes ago"
+            ,"starting from 2 hours until 30 minutes"
+            ,"starting from 2 hours ago until 30 minutes ago"
+            ,"starting from 30 minutes ago until 2 hours ago"
+            ,"from 30 minutes ago to 2 hours ago "
+    }) {
+      Window w = WindowProcessor.process(text);
+    /*
+    A dense window starting 2 hour ago and continuing until 30 minutes ago
+     */
+      Date now = new Date();
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(1, intervals.size());
+      assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(2), intervals.get(0).getMinimum());
+      assertEquals(now.getTime() - TimeUnit.MINUTES.toMillis(30), intervals.get(0).getMaximum());
+    }
+  }
+
+  @Test
+  public void testSparse() {
+    for(String text : new String[] {
+      "30 minute window every 1 hour from 2 hours ago to 30 minutes ago",
+      "30 minute window every 1 hour starting from 2 hours ago to 30 minutes ago",
+      "30 minute window every 1 hour starting from 2 hours ago until 30 minutes ago",
+      "30 minute window for every 1 hour starting from 2 hours ago until 30 minutes ago",
+    })
+    {
+      Window w = WindowProcessor.process(text);
+    /*
+    A window size of 30 minutes
+    Starting 2 hour ago and continuing until 30 minutes ago
+    window 1: ( now - 2 hour, now - 2 hour + 30 minutes)
+    window 2: (now - 1 hour, now - 1 hour + 30 minutes)
+     */
+      Date now = new Date();
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(2, intervals.size());
+      assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(2), intervals.get(0).getMinimum());
+      assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(2) + TimeUnit.MINUTES.toMillis(30),
intervals.get(0).getMaximum());
+      assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(1), intervals.get(1).getMinimum());
+      assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(1) + TimeUnit.MINUTES.toMillis(30),
intervals.get(1).getMaximum());
+    }
+  }
+
+
+  @Test
+  public void testRepeatTilNow() {
+    Window w = WindowProcessor.process("30 minute window every 1 hour from 3 hours ago");
+    /*
+    A window size of 30 minutes
+    Starting 3 hours ago and continuing until now
+    window 1: ( now - 3 hour, now - 3 hour + 30 minutes)
+    window 2: ( now - 2 hour, now - 2 hour + 30 minutes)
+    window 3: ( now - 1 hour, now - 1 hour + 30 minutes)
+     */
+    Date now = new Date();
+    List<Range<Long>> intervals = w.toIntervals(now.getTime());
+    Assert.assertEquals(3, intervals.size());
+
+    assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(3), intervals.get(0).getMinimum());
+    assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(3) + TimeUnit.MINUTES.toMillis(30),
intervals.get(0).getMaximum());
+
+    assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(2), intervals.get(1).getMinimum());
+    assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(2) + TimeUnit.MINUTES.toMillis(30),
intervals.get(1).getMaximum());
+
+    assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(1), intervals.get(2).getMinimum());
+    assertEquals(now.getTime() - TimeUnit.HOURS.toMillis(1) + TimeUnit.MINUTES.toMillis(30),
intervals.get(2).getMaximum());
+  }
+
+  @Test
+  public void testRepeatWithInclusions() {
+    {
+      Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago
including tuesdays");
+    /*
+    A window size of 30 minutes
+    Starting 14 days ago  and continuing until now
+    Gotta be 2 tuesdays in 14 days.
+     */
+      Date now = new Date();
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(2, intervals.size());
+    }
+    {
+      Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago
including this day of the week");
+    /*
+    A window size of 30 minutes
+    Starting 14 days ago  and continuing until now
+    Gotta be 2 days with the same dow in 14 days.
+     */
+      Date now = new Date();
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(2, intervals.size());
+    }
+    {
+      Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago");
+    /*
+    A window size of 30 minutes
+    Starting 14 days ago  and continuing until now
+    Gotta be 14 intervals in 14 days.
+     */
+      Date now = new Date();
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(14, intervals.size());
+    }
+  }
+
+
+  @Test
+  public void testRepeatWithConflictingExclusionInclusion() throws ParseException {
+    Window w = WindowProcessor.process("30 minute window every 24 hours from 7 days ago including
saturdays excluding weekends");
+
+    Date now = new Date();
+    List<Range<Long>> intervals = w.toIntervals(now.getTime());
+    Assert.assertEquals(0, intervals.size());
+  }
+
+  @Test
+  public void testRepeatWithWeekendExclusion() throws ParseException {
+    Window w = WindowProcessor.process("30 minute window every 24 hours from 7 days ago excluding
weekends");
+
+    Date now = new Date();
+    List<Range<Long>> intervals = w.toIntervals(now.getTime());
+    Assert.assertEquals(5, intervals.size());
+  }
+
+  @Test
+  public void testRepeatWithInclusionExclusion() throws ParseException {
+    Window w = WindowProcessor.process("30 minute window every 24 hours from 7 days ago including
holidays:us excluding weekends");
+
+    SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
+    Date now = sdf.parse("2017/12/26 12:00");
+    List<Range<Long>> intervals = w.toIntervals(now.getTime());
+    Assert.assertEquals(1, intervals.size());
+  }
+
+  @Test
+  public void testManyMoonsAgo() throws ParseException {
+    {
+      Window w = WindowProcessor.process("1 hour window every 24 hours starting from 56 days
ago");
+
+      Date now = new Date();
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(56, intervals.size());
+    }
+    {
+      Window w = WindowProcessor.process("1 hour window every 24 hours starting from 56 days
ago including this day of the week");
+
+      Date now = new Date();
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(8, intervals.size());
+    }
+  }
+
+  @Test
+  public void testRepeatWithWeekdayExclusion() throws ParseException {
+    Window w = WindowProcessor.process("30 minute window every 24 hours from 7 days ago excluding
weekdays");
+
+    Date now = new Date();
+    List<Range<Long>> intervals = w.toIntervals(now.getTime());
+    Assert.assertEquals(2, intervals.size());
+  }
+
+  @Test
+  public void testRepeatWithHolidayExclusion() throws ParseException {
+    {
+      Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago
excluding holidays:us");
+      SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
+      Date now = sdf.parse("2017/12/26 12:00");
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(13, intervals.size());
+    }
+    {
+      Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago
excluding holidays:us:nyc");
+      SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
+      Date now = sdf.parse("2017/12/26 12:00");
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(13, intervals.size());
+    }
+    {
+      Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago
excluding holidays:us");
+      SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
+      Date now = sdf.parse("2017/08/26 12:00");
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(14, intervals.size());
+    }
+  }
+
+  @Test
+  public void testDateDaySpecifier() throws ParseException {
+    for(String text : new String[] {
+        "30 minute window every 24 hours from 14 days ago including date:20171225:yyyyMMdd",
+        "30 minute window every 24 hours from 14 days ago including date:2017-12-25:yyyy-MM-dd",
+        "30 minute window every 24 hours from 14 days ago including date:2017/12/25",
+      })
+    {
+      Window w = WindowProcessor.process(text);
+      SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
+      Date now = sdf.parse("2017/12/26 12:00");
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(1, intervals.size());
+      Date includedDate = new Date(intervals.get(0).getMinimum());
+      SimpleDateFormat equalityFormat = new SimpleDateFormat("yyyyMMdd");
+      Assert.assertEquals("20171225", equalityFormat.format(includedDate));
+    }
+    {
+      Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago
excluding date:2017/12/25");
+      SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
+      Date now = sdf.parse("2017/12/26 12:00");
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(13, intervals.size());
+    }
+    {
+      Window w = WindowProcessor.process("30 minute window every 24 hours from 14 days ago
including date:2017/12/25, date:2017/12/24");
+      SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm");
+      Date now = sdf.parse("2017/12/26 12:00");
+      List<Range<Long>> intervals = w.toIntervals(now.getTime());
+      Assert.assertEquals(2, intervals.size());
+      {
+        Date includedDate = new Date(intervals.get(0).getMinimum());
+        SimpleDateFormat equalityFormat = new SimpleDateFormat("yyyyMMdd");
+        Assert.assertEquals("20171224", equalityFormat.format(includedDate));
+      }
+      {
+        Date includedDate = new Date(intervals.get(1).getMinimum());
+        SimpleDateFormat equalityFormat = new SimpleDateFormat("yyyyMMdd");
+        Assert.assertEquals("20171225", equalityFormat.format(includedDate));
+      }
+    }
+  }
+
+  @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+  public void testWithInvalidDaySpecifier() throws ParseException {
+    WindowProcessor.process("30 minute window every 24 hours from 14 days ago excluding hoolidays:us");
+  }
+
+  @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+  public void testWithInvalidTimeUnit() throws ParseException {
+    WindowProcessor.process("30 minute window every 24 months from 14 days ago");
+  }
+
+  @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+  public void testWithInvalidWindowUnit() throws ParseException {
+    WindowProcessor.process("30 minuete window every 24 hours from 14 days ago");
+  }
+
+  @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+  public void testWithInvalidTimeNumber() throws ParseException {
+    WindowProcessor.process("30p minute window every 24 hours from 14 days ago");
+  }
+
+  @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+  public void testInvalidDaySpecifier() throws ParseException {
+    WindowProcessor.process("30 minute window every 14 hours from 14 days ago including date");
+  }
+
+  private static void assertEquals(long expected, long actual) {
+    long diff = expected - actual;
+    long diffInMinutes = TimeUnit.MILLISECONDS.toMinutes(diff);
+    String message =  expected + " - " + actual + " = " + diffInMinutes + " minutes off.";
+    Assert.assertEquals(message, expected, actual);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/84d34719/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
index f916d65..2f7f356 100644
--- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
+++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfilePeriod.java
@@ -111,12 +111,12 @@ public class ProfilePeriod {
   }
 
   public static <T> List<T> visitPeriods(long startEpochMillis
-                                           , long endEpochMillis
-                                           , long duration
-                                           , TimeUnit units
-                                           , Optional<Predicate<ProfilePeriod>>
inclusionPredicate
-                                           , Function<ProfilePeriod,T> transformation
-                                           )
+                                        , long endEpochMillis
+                                        , long duration
+                                        , TimeUnit units
+                                        , Optional<Predicate<ProfilePeriod>>
inclusionPredicate
+                                        , Function<ProfilePeriod,T> transformation
+                                        )
   {
     ProfilePeriod period = new ProfilePeriod(startEpochMillis, duration, units);
     List<T> ret = new ArrayList<>();

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/84d34719/metron-platform/metron-common/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md
index 067bf8d..4ffb8f1 100644
--- a/metron-platform/metron-common/README.md
+++ b/metron-platform/metron-common/README.md
@@ -122,6 +122,7 @@ The `!=` operator is the negation of the above.
 | [ `MONTH`](#month)                                                                    
            |
 | [ `PROFILE_GET`](#profile_get)                                                        
            |
 | [ `PROFILE_FIXED`](#profile_fixed)                                                    
                |
+| [ `PROFILE_WINDOW`](#profile_window)                                                  
                  |
 | [ `PROTOCOL_TO_NAME`](#protocol_to_name)                                              
            |
 | [ `REGEXP_MATCH`](#regexp_match)                                                      
            |
 | [ `SPLIT`](#split)                                                                    
            |
@@ -450,6 +451,14 @@ The `!=` operator is the negation of the above.
     * config_overrides - Optional - Map (in curly braces) of name:value pairs, each overriding
the global config parameter of the same name. Default is the empty Map, meaning no overrides.
   * Returns: The selected profile measurement timestamps.  These are ProfilePeriod objects.
 
+### `PROFILE_WINDOW`
+  * Description: The profiler periods associated with a window selector statement from an
optional reference timestamp.
+  * Input:
+    * windowSelector - The statement specifying the window to select.
+    * now - Optional - The timestamp to use for now.
+    * config_overrides - Optional - Map (in curly braces) of name:value pairs, each overriding
the global config parameter of the same name. Default is the empty Map, meaning no overrides.
+  * Returns: The selected profile measurement periods.  These are ProfilePeriod objects.
+
 ### `PROTOCOL_TO_NAME`
   * Description: Converts the IANA protocol number to the protocol name
   * Input:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/84d34719/metron-platform/metron-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/pom.xml b/metron-platform/metron-common/pom.xml
index 9c9e761..5d4c2fd 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -27,7 +27,6 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <commons.config.version>1.10</commons.config.version>
-        <antlr.version>4.5</antlr.version>
     </properties>
     <repositories>
         <repository>
@@ -65,7 +64,7 @@
         <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
-            <version>${antlr.version}</version>
+            <version>${global_antlr_version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.storm</groupId>
@@ -376,7 +375,7 @@
             <plugin>
                 <groupId>org.antlr</groupId>
                 <artifactId>antlr4-maven-plugin</artifactId>
-                <version>${antlr.version}</version>
+                <version>${global_antlr_version}</version>
                 <configuration>
                   <outputDirectory>${basedir}/src/main/java</outputDirectory>
                 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/84d34719/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/GrammarUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/GrammarUtils.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/GrammarUtils.java
new file mode 100644
index 0000000..e65af94
--- /dev/null
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/GrammarUtils.java
@@ -0,0 +1,133 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.metron.common.dsl;
+
+import org.antlr.v4.runtime.tree.ParseTree;
+import org.antlr.v4.runtime.Token;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class GrammarUtils {
+
+  public static String toSyntaxTree(ParseTree tree) {
+    return new AST(tree).toString();
+  }
+
+  /**
+   * This is a utility class to walk the parse tree for an antlr ParseTree
+   */
+  private static class AST {
+
+    private final Object payload;
+    private final List<AST> children;
+
+    public AST(ParseTree tree) {
+      this(null, tree);
+    }
+
+    private AST(AST ast, ParseTree tree) {
+      this(ast, tree, new ArrayList<>());
+    }
+
+    private AST(AST parent, ParseTree tree, List<AST> children) {
+      this.payload = getPayload(tree);
+      this.children = children;
+      if (parent == null) {
+        walk(tree, this);
+      }
+      else {
+        parent.children.add(this);
+      }
+    }
+
+    private Object getPayload(ParseTree tree) {
+      if (tree.getChildCount() == 0) {
+        return tree.getPayload();
+      }
+      else {
+        String ruleName = tree.getClass().getSimpleName().replace("Context", "");
+        return Character.toLowerCase(ruleName.charAt(0)) + ruleName.substring(1);
+      }
+    }
+
+    private static void walk(ParseTree tree, AST ast) {
+      if (tree.getChildCount() == 0) {
+        new AST(ast, tree);
+      }
+      else if (tree.getChildCount() == 1) {
+        walk(tree.getChild(0), ast);
+      }
+      else if (tree.getChildCount() > 1) {
+        for (int i = 0; i < tree.getChildCount(); i++) {
+          AST temp = new AST(ast, tree.getChild(i));
+          if (!(temp.payload instanceof Token)) {
+            walk(tree.getChild(i), temp);
+          }
+        }
+      }
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder builder = new StringBuilder();
+      AST ast = this;
+      List<AST> firstStack = new ArrayList<>();
+      firstStack.add(ast);
+      List<List<AST>> childListStack = new ArrayList<>();
+      childListStack.add(firstStack);
+      while (!childListStack.isEmpty()) {
+        List<AST> childStack = childListStack.get(childListStack.size() - 1);
+        if (childStack.isEmpty()) {
+          childListStack.remove(childListStack.size() - 1);
+        }
+        else {
+          ast = childStack.remove(0);
+          String caption;
+          if (ast.payload instanceof Token) {
+            Token token = (Token) ast.payload;
+            caption = String.format("TOKEN[type: %s, text: %s]",
+                    token.getType(), token.getText().replace("\n", "\\n"));
+          }
+          else {
+            caption = String.valueOf(ast.payload);
+          }
+          String indent = "";
+          for (int i = 0; i < childListStack.size() - 1; i++) {
+            indent += (childListStack.get(i).size() > 0) ? "|  " : "   ";
+          }
+          builder.append(indent)
+                  .append(childStack.isEmpty() ? "'- " : "|- ")
+                  .append(caption)
+                  .append("\n");
+
+          if (ast.children.size() > 0) {
+            List<AST> children = new ArrayList<>();
+            for (int i = 0; i < ast.children.size(); i++) {
+              children.add(ast.children.get(i));
+            }
+            childListStack.add(children);
+          }
+        }
+      }
+      return builder.toString();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/84d34719/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Token.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Token.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Token.java
index f2c56a5..4c94be0 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Token.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/Token.java
@@ -34,7 +34,7 @@ public class Token<T> {
 
   @Override
   public String toString() {
-    return value.toString();
+    return "" + value;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/84d34719/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 04c3954..db28adb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -71,6 +71,7 @@
         <base_hbase_version>1.1.1</base_hbase_version>
         <base_flume_version>1.5.2</base_flume_version>
         <!-- full dependency versions -->
+        <global_antlr_version>4.5</global_antlr_version>
         <global_opencsv_version>3.7</global_opencsv_version>
         <global_curator_version>2.7.1</global_curator_version>
         <global_storm_version>${base_storm_version}</global_storm_version>



Mime
View raw message