Return-Path: X-Original-To: apmail-olingo-commits-archive@minotaur.apache.org Delivered-To: apmail-olingo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0AB1A17FED for ; Thu, 19 Nov 2015 18:57:36 +0000 (UTC) Received: (qmail 12209 invoked by uid 500); 19 Nov 2015 18:57:35 -0000 Delivered-To: apmail-olingo-commits-archive@olingo.apache.org Received: (qmail 12189 invoked by uid 500); 19 Nov 2015 18:57:35 -0000 Mailing-List: contact commits-help@olingo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@olingo.apache.org Delivered-To: mailing list commits@olingo.apache.org Received: (qmail 12180 invoked by uid 99); 19 Nov 2015 18:57:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 19 Nov 2015 18:57:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A6DBDE17DA; Thu, 19 Nov 2015 18:57:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mibo@apache.org To: commits@olingo.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: olingo-odata4 git commit: [OLINGO-568] More tests and fixes Date: Thu, 19 Nov 2015 18:57:35 +0000 (UTC) Repository: olingo-odata4 Updated Branches: refs/heads/master 233ea61f3 -> 63db8b36c [OLINGO-568] More tests and fixes Project: http://git-wip-us.apache.org/repos/asf/olingo-odata4/repo Commit: http://git-wip-us.apache.org/repos/asf/olingo-odata4/commit/63db8b36 Tree: http://git-wip-us.apache.org/repos/asf/olingo-odata4/tree/63db8b36 Diff: http://git-wip-us.apache.org/repos/asf/olingo-odata4/diff/63db8b36 Branch: refs/heads/master Commit: 63db8b36c4e789a1e3fb7fb52aeb40a6e6734a1b Parents: 233ea61 Author: mibo Authored: Thu Nov 19 19:50:34 2015 +0100 Committer: mibo Committed: Thu Nov 19 19:50:34 2015 +0100 ---------------------------------------------------------------------- .../core/uri/parser/search/SearchParser.java | 7 + .../core/uri/parser/search/SearchTokenizer.java | 38 +- .../search/SearchParserAndTokenizerTest.java | 145 ++------ .../uri/parser/search/SearchParserTest.java | 1 - .../uri/parser/search/SearchTokenizerTest.java | 348 +++++++------------ 5 files changed, 199 insertions(+), 340 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/63db8b36/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java index 2723024..3e16444 100644 --- a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java +++ b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java @@ -129,6 +129,9 @@ public class SearchParser { SearchExpression se = left; if (isTerm()) { se = processTerm(); + if(isTerm()) { + se = processAnd(se); + } se = new SearchBinaryImpl(left, SearchBinaryOperatorKind.AND, se); return processSearchExpression(se); } else { @@ -154,6 +157,10 @@ public class SearchParser { if (isToken(Token.WORD) || isToken(Token.PHRASE)) { return new SearchUnaryImpl(processWordOrPhrase()); } + if(isEof()) { + throw new SearchParserException("NOT must be followed by a term.", + SearchParserException.MessageKeys.INVALID_NOT_OPERAND, "EOF"); + } throw new SearchParserException("NOT must be followed by a term not a " + token.getToken(), SearchParserException.MessageKeys.INVALID_NOT_OPERAND, token.getToken().toString()); } http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/63db8b36/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java index dd65852..ffff1b5 100644 --- a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java +++ b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java @@ -76,7 +76,7 @@ public class SearchTokenizer { } public State invalid() throws SearchTokenizerException { - throw new SearchTokenizerException("Token " + this.getToken() + " is in invalid state ", + throw new SearchTokenizerException("Token " + this.getToken() + " is in invalid state.", SearchTokenizerException.MessageKeys.INVALID_TOKEN_STATE); } @@ -428,7 +428,7 @@ public class SearchTokenizer { if(closed) { return finish(); } - return super.close(); + return invalid(); } } @@ -475,15 +475,18 @@ public class SearchTokenizer { } else if (literal.length() == 3 && isWhitespace(c)) { finish(); return new BeforePhraseOrWordRwsState(); + } else if(isWhitespace(c)) { + changeToken(Token.WORD).finish(); + return new RwsState(); } - return forbidden(c); + return new SearchWordState(this).nextChar(c); } @Override public State close() throws SearchTokenizerException { if(Token.NOT.name().equals(literal.toString())) { return finish(); } - return super.close(); + return changeToken(Token.WORD).finish(); } } @@ -504,9 +507,18 @@ public class SearchTokenizer { } else if (literal.length() == 3 && isWhitespace(c)) { finish(); return new BeforeSearchExpressionRwsState(); - } else { - return new SearchWordState(this); + } else if(isWhitespace(c)) { + changeToken(Token.WORD).finish(); + return new RwsState(); + } + return new SearchWordState(this).nextChar(c); + } + @Override + public State close() throws SearchTokenizerException { + if(Token.AND.name().equals(literal.toString())) { + return finish(); } + return changeToken(Token.WORD).finish(); } } @@ -517,7 +529,6 @@ public class SearchTokenizer { forbidden(c); } } - @Override public State nextChar(char c) throws SearchTokenizerException { if (literal.length() == 1 && (c == CHAR_R)) { @@ -525,9 +536,18 @@ public class SearchTokenizer { } else if (literal.length() == 2 && isWhitespace(c)) { finish(); return new BeforeSearchExpressionRwsState(); - } else { - return new SearchWordState(this); + } else if(isWhitespace(c)) { + changeToken(Token.WORD).finish(); + return new RwsState(); + } + return new SearchWordState(this).nextChar(c); + } + @Override + public State close() throws SearchTokenizerException { + if(Token.OR.name().equals(literal.toString())) { + return finish(); } + return changeToken(Token.WORD).finish(); } } http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/63db8b36/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserAndTokenizerTest.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserAndTokenizerTest.java b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserAndTokenizerTest.java index fa419a9..0aa79ab 100644 --- a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserAndTokenizerTest.java +++ b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserAndTokenizerTest.java @@ -18,15 +18,9 @@ */ package org.apache.olingo.server.core.uri.parser.search; -import static org.apache.olingo.server.api.uri.queryoption.search.SearchBinaryOperatorKind.AND; -import static org.apache.olingo.server.api.uri.queryoption.search.SearchBinaryOperatorKind.OR; - -import java.lang.reflect.Field; - import org.apache.olingo.server.api.ODataLibraryException; import org.apache.olingo.server.api.uri.queryoption.SearchOption; import org.apache.olingo.server.api.uri.queryoption.search.SearchExpression; -import org.apache.olingo.server.api.uri.queryoption.search.SearchUnary; import org.junit.Assert; import org.junit.Test; @@ -34,149 +28,86 @@ public class SearchParserAndTokenizerTest { @Test public void basicParsing() throws Exception { - SearchExpressionValidator.init("\"99\"") - .validate(with("99")); - SearchExpressionValidator.init("a") - .validate(with("a")); - SearchExpressionValidator.init("a AND b") - .validate(with("a", and("b"))); - SearchExpressionValidator.init("a AND b AND c") - .validate("{{'a' AND 'b'} AND 'c'}"); - SearchExpressionValidator.init("a OR b") - .validate(with("a", or("b"))); - SearchExpressionValidator.init("a OR b OR c") - .validate(with("a", or("b", or("c")))); + assertQuery("\"99\"").resultsIn("'99'"); + assertQuery("a").resultsIn("'a'"); + assertQuery("a AND b").resultsIn("{'a' AND 'b'}"); + assertQuery("a AND b AND c").resultsIn("{{'a' AND 'b'} AND 'c'}"); + assertQuery("a OR b").resultsIn("{'a' OR 'b'}"); + assertQuery("a OR b OR c").resultsIn("{'a' OR {'b' OR 'c'}}"); } @Test public void mixedParsing() throws Exception { - SearchExpressionValidator.init("a AND b OR c") - .validate("{{'a' AND 'b'} OR 'c'}"); - SearchExpressionValidator.init("a OR b AND c") - .validate("{'a' OR {'b' AND 'c'}}"); + assertQuery("a AND b OR c").resultsIn("{{'a' AND 'b'} OR 'c'}"); + assertQuery("a OR b AND c").resultsIn("{'a' OR {'b' AND 'c'}}"); } @Test public void notParsing() throws Exception { - SearchExpressionValidator.init("NOT a AND b OR c") - .validate("{{{NOT 'a'} AND 'b'} OR 'c'}"); - SearchExpressionValidator.init("a OR b AND NOT c") - .validate("{'a' OR {'b' AND {NOT 'c'}}}"); + assertQuery("NOT a AND b OR c").resultsIn("{{{NOT 'a'} AND 'b'} OR 'c'}"); + assertQuery("a OR b AND NOT c").resultsIn("{'a' OR {'b' AND {NOT 'c'}}}"); } @Test public void parenthesesParsing() throws Exception { - SearchExpressionValidator.init("a AND (b OR c)") - .validate("{'a' AND {'b' OR 'c'}}"); - SearchExpressionValidator.init("(a OR b) AND NOT c") - .validate("{{'a' OR 'b'} AND {NOT 'c'}}"); + assertQuery("a AND (b OR c)").resultsIn("{'a' AND {'b' OR 'c'}}"); + assertQuery("(a OR b) AND NOT c").resultsIn("{{'a' OR 'b'} AND {NOT 'c'}}"); } @Test - public void invalidSearchQuery() throws Exception { - SearchExpressionValidator.init("99").validate(SearchParserException.class, - SearchParserException.MessageKeys.TOKENIZER_EXCEPTION); - } - - private static SearchExpression with(String term) { - return new SearchTermImpl(term); - } - - private static SearchExpression with(String left, SearchExpression right) { - setLeftField(left, right); - return right; - } - - private static SearchUnary with(SearchUnary unary) { - return unary; - } - - private static SearchExpression or(String left, SearchExpression right) { - SearchExpression or = or(right); - setLeftField(left, right); - return or; - } - - private static SearchExpression and(String left, SearchExpression right) { - SearchExpression and = and(right); - setLeftField(left, right); - return and; - } - - private static SearchExpression or(SearchExpression right) { - return new SearchBinaryImpl(null, OR, right); - } - - private static SearchExpression and(SearchExpression right) { - return new SearchBinaryImpl(null, AND, right); - } - - private static SearchExpression and(String right) { - return and(new SearchTermImpl(right)); - } - - private static SearchExpression or(String right) { - return or(new SearchTermImpl(right)); + public void parseImplicitAnd() throws Exception { + assertQuery("a b").resultsIn("{'a' AND 'b'}"); + assertQuery("a b c").resultsIn("{'a' AND {'b' AND 'c'}}"); + assertQuery("a and b").resultsIn("{'a' AND {'and' AND 'b'}}"); + assertQuery("a b OR c").resultsIn("{{'a' AND 'b'} OR 'c'}"); + assertQuery("a \"bc123\" OR c").resultsIn("{{'a' AND 'bc123'} OR 'c'}"); + assertQuery("(a OR x) bc c").resultsIn("{{'a' OR 'x'} AND {'bc' AND 'c'}}"); + assertQuery("one ((a OR x) bc c)").resultsIn("{'one' AND {{'a' OR 'x'} AND {'bc' AND 'c'}}}"); } - private static SearchUnary not(String term) { - return new SearchUnaryImpl(new SearchTermImpl(term)); + @Test + public void invalidSearchQuery() throws Exception { + assertQuery("99").resultsIn(SearchParserException.MessageKeys.TOKENIZER_EXCEPTION); + assertQuery("NOT").resultsIn(SearchParserException.MessageKeys.INVALID_NOT_OPERAND); + assertQuery("AND").resultsIn(SearchParserException.MessageKeys.INVALID_BINARY_OPERATOR_POSITION); + assertQuery("OR").resultsIn(SearchParserException.MessageKeys.INVALID_BINARY_OPERATOR_POSITION); } - private static void setLeftField(String left, SearchExpression se) { - try { - Field field = null; - if (se instanceof SearchUnaryImpl) { - field = SearchBinaryImpl.class.getDeclaredField("operand"); - } else if (se instanceof SearchBinaryImpl) { - field = SearchBinaryImpl.class.getDeclaredField("left"); - } else { - Assert.fail("Unexpected exception: " + se.getClass()); - } - field.setAccessible(true); - field.set(se, new SearchTermImpl(left)); - } catch (Exception e) { - Assert.fail("Unexpected exception: " + e.getClass()); - } + private static Validator assertQuery(String searchQuery) { + return Validator.init(searchQuery); } - private static class SearchExpressionValidator { + private static class Validator { private boolean log; private final String searchQuery; - private SearchExpressionValidator(String searchQuery) { + private Validator(String searchQuery) { this.searchQuery = searchQuery; } - private static SearchExpressionValidator init(String searchQuery) { - return new SearchExpressionValidator(searchQuery); + private static Validator init(String searchQuery) { + return new Validator(searchQuery); } @SuppressWarnings("unused") - private SearchExpressionValidator enableLogging() { + private Validator withLogging() { log = true; return this; } - private void validate(Class exception, ODataLibraryException.MessageKey key) + private void resultsIn(ODataLibraryException.MessageKey key) throws SearchTokenizerException { try { - validate(searchQuery); + resultsIn(searchQuery); } catch (ODataLibraryException e) { - Assert.assertEquals(exception, e.getClass()); + Assert.assertEquals(SearchParserException.class, e.getClass()); Assert.assertEquals(key, e.getMessageKey()); return; } - Assert.fail("Expected exception " + exception.getClass().getSimpleName() + " was not thrown."); - } - - private void validate(SearchExpression expectedSearchExpression) throws SearchTokenizerException, - SearchParserException { - final SearchExpression searchExpression = getSearchExpression(); - Assert.assertEquals(expectedSearchExpression.toString(), searchExpression.toString()); + Assert.fail("SearchParserException with message key " + key.getKey() + " was not thrown."); } - private void validate(String expectedSearchExpression) throws SearchTokenizerException, SearchParserException { + private void resultsIn(String expectedSearchExpression) throws SearchTokenizerException, SearchParserException { final SearchExpression searchExpression = getSearchExpression(); Assert.assertEquals(expectedSearchExpression, searchExpression.toString()); } http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/63db8b36/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java index ee10e1a..5d3f2dc 100644 --- a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java +++ b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java @@ -46,7 +46,6 @@ public class SearchParserTest extends SearchParser { se = run(Token.PHRASE); assertEquals("'phrase1'", se.toString()); assertTrue(se.isSearchTerm()); - // TODO: Check if quotation marks should be part of the string we deliver assertEquals("phrase1", se.asSearchTerm().getSearchTerm()); } http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/63db8b36/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java index c5a27f2..133bb2e 100644 --- a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java +++ b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java @@ -38,52 +38,28 @@ public class SearchTokenizerTest { @Test public void parseBasics() throws Exception { - SearchTokenizer tokenizer = new SearchTokenizer(); - List result; - - // - result = tokenizer.tokenize("abc"); - Assert.assertNotNull(result); - - Assert.assertEquals(WORD, result.get(0).getToken()); - - result = tokenizer.tokenize("NOT abc"); - Assert.assertNotNull(result); - - Assert.assertEquals(NOT, result.get(0).getToken()); - Assert.assertEquals(WORD, result.get(1).getToken()); - - result = tokenizer.tokenize("(abc)"); - Assert.assertNotNull(result); - - Assert.assertEquals(OPEN, result.get(0).getToken()); - Assert.assertEquals(WORD, result.get(1).getToken()); - Assert.assertEquals(CLOSE, result.get(2).getToken()); - - result = tokenizer.tokenize("((abc))"); - Assert.assertNotNull(result); - - Assert.assertEquals(OPEN, result.get(0).getToken()); - Assert.assertEquals(WORD, result.get(2).getToken()); - Assert.assertEquals(CLOSE, result.get(4).getToken()); + assertQuery("abd").resultsIn(WORD); + assertQuery("NOT abc").resultsIn(NOT, WORD); + assertQuery("(abc)").resultsIn(OPEN, WORD, CLOSE); + assertQuery("((abc))").resultsIn(OPEN, OPEN, WORD, CLOSE, CLOSE); } @Test public void parseWords() throws Exception { - SearchTokenizer tokenizer = new SearchTokenizer(); - List result; - - // - result = tokenizer.tokenize("abc"); - Assert.assertNotNull(result); - - Assert.assertEquals(WORD, result.get(0).getToken()); + assertQuery("somesimpleword").resultsIn(WORD); + assertQuery("anotherWord\u1234").resultsIn(WORD); + // special + assertQuery("NO").resultsIn(word("NO")); + assertQuery("N").resultsIn(word("N")); + assertQuery("A").resultsIn(word("A")); + assertQuery("AN").resultsIn(word("AN")); + assertQuery("O").resultsIn(word("O")); + // invalid + assertQuery("notAw0rd").resultsIn(SearchTokenizerException.MessageKeys.FORBIDDEN_CHARACTER); + } - // - result = tokenizer.tokenize("anotherWord\u1234"); - Assert.assertNotNull(result); - - Assert.assertEquals(WORD, result.get(0).getToken()); + private Validator.Tuple word(String literal) { + return Validator.tuple(WORD, literal); } @Test @@ -91,7 +67,7 @@ public class SearchTokenizerTest { SearchTokenizer tokenizer = new SearchTokenizer(); List result; - TokenizerValidator.init("abc AND \"x-y_z\" AND olingo").validate(); + assertQuery("abc AND \"x-y_z\" AND olingo"); // result = tokenizer.tokenize("\"abc\""); @@ -113,7 +89,7 @@ public class SearchTokenizerTest { Assert.assertEquals(PHRASE, result.get(0).getToken()); Assert.assertEquals("\"99_88.\"", result.get(0).getLiteral()); - TokenizerValidator.init("abc or \"xyz\"").validate(WORD, WORD, PHRASE); + assertQuery("abc or \"xyz\"").resultsIn(WORD, WORD, PHRASE); } /** @@ -124,165 +100,95 @@ public class SearchTokenizerTest { @Ignore("Test must be moved to SearchParserTest and SearchParserAndTokenizerTest") public void parsePhraseAbnfTestcases() throws Exception { // - TokenizerValidator.init("\"blue%20green\"").validate(); + assertQuery("\"blue%20green\""); // - TokenizerValidator.init("\"blue%20green%22").validate(); + assertQuery("\"blue%20green%22"); // // $search="blue\"green" - TokenizerValidator.init("\"blue\\\"green\"").validate(); + assertQuery("\"blue\\\"green\""); // // $search="blue\\green" - TokenizerValidator.init("\"blue\\\\green\"").validate(); + assertQuery("\"blue\\\\green\""); // - TokenizerValidator.init("\"blue\"green\"").validate(); + assertQuery("\"blue\"green\""); // - TokenizerValidator.init("\"blue%22green\"").validate(); + assertQuery("\"blue%22green\""); // // $search=blue green -// SearchValidator.init("\"blue%20green\"").validate(); +// SearchassertQuery("\"blue%20green\"").resultsIn(); // -// SearchValidator.init("blue%20green").validate(); +// SearchassertQuery("blue%20green").resultsIn(); } @Test public void parseNot() throws Exception { - SearchTokenizer tokenizer = new SearchTokenizer(); - List result; - - result = tokenizer.tokenize("NOT abc"); - Assert.assertNotNull(result); - - Assert.assertEquals(NOT, result.get(0).getToken()); - Assert.assertEquals(WORD, result.get(1).getToken()); - - TokenizerValidator.init("not abc").addExpected(WORD, WORD).validate(); - TokenizerValidator.init("NOT abc").addExpected(NOT, WORD).validate(); - TokenizerValidator.init("NOT \"abc\"").addExpected(NOT, PHRASE).validate(); - TokenizerValidator.init("NOT (sdf)").validate(SearchTokenizerException.class); + assertQuery("NOT").resultsIn(NOT); + assertQuery(" NOT ").resultsIn(NOT); + assertQuery("NOT abc").resultsIn(NOT, WORD); + assertQuery("not abc").resultsIn(WORD, WORD); + assertQuery("NOT abc").resultsIn(NOT, WORD); + assertQuery("NOT \"abc\"").resultsIn(NOT, PHRASE); + assertQuery("NObody").resultsIn(WORD); + assertQuery("Nobody").resultsIn(WORD); + assertQuery("NOT (sdf)").resultsIn(SearchTokenizerException.MessageKeys.FORBIDDEN_CHARACTER); } @Test public void parseOr() throws Exception { - SearchTokenizer tokenizer = new SearchTokenizer(); - List result; - - result = tokenizer.tokenize("abc OR xyz"); - Assert.assertNotNull(result); - - Assert.assertEquals(WORD, result.get(0).getToken()); - Assert.assertEquals(OR, result.get(1).getToken()); - Assert.assertEquals(WORD, result.get(2).getToken()); - - result = tokenizer.tokenize("abc OR xyz OR olingo"); - Assert.assertNotNull(result); - - Assert.assertEquals(WORD, result.get(0).getToken()); - Assert.assertEquals(OR, result.get(1).getToken()); - Assert.assertEquals(WORD, result.get(2).getToken()); - Assert.assertEquals(OR, result.get(3).getToken()); - Assert.assertEquals(WORD, result.get(4).getToken()); - - TokenizerValidator.init("abc or xyz").addExpected(WORD, WORD, WORD).validate(); + assertQuery("OR").resultsIn(OR); + assertQuery(" OR ").resultsIn(OR); + assertQuery("OR xyz").resultsIn(OR, WORD); + assertQuery("abc OR xyz").resultsIn(WORD, OR, WORD); + assertQuery("abc OR xyz OR olingo").resultsIn(WORD, OR, WORD, OR, WORD); + assertQuery("abc or xyz").addExpected(WORD, WORD, WORD); } @Test public void parseImplicitAnd() throws SearchTokenizerException { - TokenizerValidator.init("a b").addExpected(WORD, WORD).validate(); - TokenizerValidator.init("a b OR c").addExpected(WORD, WORD, OR, WORD).validate(); - TokenizerValidator.init("a bc OR c").addExpected(WORD, WORD, OR, WORD).validate(); - TokenizerValidator.init("a bc c").addExpected(WORD, WORD, WORD).validate(); - TokenizerValidator.init("(a OR x) bc c").addExpected(OPEN, WORD, OR, WORD, CLOSE, WORD, WORD).validate(); + assertQuery("a b").resultsIn(WORD, WORD); + assertQuery("a b OR c").resultsIn(WORD, WORD, OR, WORD); + assertQuery("a bc OR c").resultsIn(WORD, WORD, OR, WORD); + assertQuery("a bc c").resultsIn(WORD, WORD, WORD); + assertQuery("(a OR x) bc c").resultsIn(OPEN, WORD, OR, WORD, CLOSE, WORD, WORD); } @Test public void parseAnd() throws Exception { - SearchTokenizer tokenizer = new SearchTokenizer(); - List result; - - result = tokenizer.tokenize("abc AND xyz"); - Assert.assertNotNull(result); - - Assert.assertEquals(WORD, result.get(0).getToken()); - Assert.assertEquals(AND, result.get(1).getToken()); - Assert.assertEquals(WORD, result.get(2).getToken()); + assertQuery("AND").resultsIn(AND); + assertQuery(" AND ").resultsIn(AND); + assertQuery("abc AND xyz").resultsIn(WORD, AND, WORD); // no lower case allowed for AND - result = tokenizer.tokenize("abc and xyz"); - Assert.assertNotNull(result); - Assert.assertEquals(3, result.size()); - - Assert.assertEquals(WORD, result.get(0).getToken()); - Assert.assertEquals(WORD, result.get(1).getToken()); - Assert.assertEquals(WORD, result.get(2).getToken()); - - // implicit AND - result = tokenizer.tokenize("abc xyz"); - Assert.assertNotNull(result); - - Assert.assertEquals(WORD, result.get(0).getToken()); - Assert.assertEquals(WORD, result.get(1).getToken()); - - result = tokenizer.tokenize("abc AND xyz AND olingo"); - Assert.assertNotNull(result); - - Assert.assertEquals(WORD, result.get(0).getToken()); - Assert.assertEquals(AND, result.get(1).getToken()); - Assert.assertEquals(WORD, result.get(2).getToken()); - Assert.assertEquals(AND, result.get(3).getToken()); - Assert.assertEquals(WORD, result.get(4).getToken()); - - result = tokenizer.tokenize("abc AND \"x-y_z\" AND olingo"); - Assert.assertNotNull(result); - - Assert.assertEquals(WORD, result.get(0).getToken()); - Assert.assertEquals(AND, result.get(1).getToken()); - Assert.assertEquals(PHRASE, result.get(2).getToken()); - Assert.assertEquals("\"x-y_z\"", result.get(2).getLiteral()); - Assert.assertEquals(AND, result.get(3).getToken()); - Assert.assertEquals(WORD, result.get(4).getToken()); + assertQuery("abc and xyz").resultsIn(WORD, WORD, WORD); + // implicit AND is handled by parser (and not tokenizer) + assertQuery("abc xyz").resultsIn(WORD, WORD); + assertQuery("abc AND xyz AND olingo").resultsIn(WORD, AND, WORD, AND, WORD); + assertQuery("abc AND \"x-y_z\" AND olingo") + .resultsIn(WORD, AND, PHRASE, AND, WORD); } @Test public void parseAndOr() throws Exception { - SearchTokenizer tokenizer = new SearchTokenizer(); - List result; - - result = tokenizer.tokenize("abc AND xyz OR olingo"); - Assert.assertNotNull(result); - - Assert.assertEquals(WORD, result.get(0).getToken()); - Assert.assertEquals(AND, result.get(1).getToken()); - Assert.assertEquals(WORD, result.get(2).getToken()); - Assert.assertEquals(OR, result.get(3).getToken()); - Assert.assertEquals(WORD, result.get(4).getToken()); - - TokenizerValidator.init("abc AND ANDsomething") - .addExpected(WORD, AND, WORD).validate(); + assertQuery("OR AND ").resultsIn(OR, AND); + assertQuery("abc AND xyz OR olingo").resultsIn(WORD, AND, WORD, OR, WORD); + assertQuery("abc AND ANDsomething").addExpected(WORD, AND, WORD); } @Test public void parseCombinations() throws Exception { - SearchTokenizer tokenizer = new SearchTokenizer(); - List result; + assertQuery("word O NO").resultsIn(word("word"), word("O"), word("NO")); + assertQuery("O AN NO").resultsIn(word("O"), word("AN"), word("NO")); + assertQuery("NO AN O").resultsIn(word("NO"), word("AN"), word("O")); + assertQuery("N A O").resultsIn(word("N"), word("A"), word("O")); + assertQuery("abc AND NOT xyz OR olingo").resultsIn(WORD, AND, NOT, WORD, OR, WORD); - result = tokenizer.tokenize("abc AND NOT xyz OR olingo"); - Assert.assertNotNull(result); - - Iterator it = result.iterator(); - Assert.assertEquals(WORD, it.next().getToken()); - Assert.assertEquals(AND, it.next().getToken()); - Assert.assertEquals(NOT, it.next().getToken()); - Assert.assertEquals(WORD, it.next().getToken()); - Assert.assertEquals(OR, it.next().getToken()); - Assert.assertEquals(WORD, it.next().getToken()); - - TokenizerValidator.init("foo AND bar OR foo AND baz OR that AND bar OR that AND baz") + assertQuery("foo AND bar OR foo AND baz OR that AND bar OR that AND baz") .addExpected(WORD, "foo").addExpected(AND) .addExpected(WORD, "bar").addExpected(OR) .addExpected(WORD, "foo").addExpected(AND) @@ -294,7 +200,7 @@ public class SearchTokenizerTest { .validate(); - TokenizerValidator.init("(foo OR that) AND (bar OR baz)") + assertQuery("(foo OR that) AND (bar OR baz)") .addExpected(OPEN) .addExpected(WORD, "foo").addExpected(OR).addExpected(WORD, "that") .addExpected(CLOSE).addExpected(AND).addExpected(OPEN) @@ -306,47 +212,21 @@ public class SearchTokenizerTest { @Test public void parseSpecial() throws Exception { - SearchTokenizer tokenizer = new SearchTokenizer(); - List result; - Iterator it; - - result = tokenizer.tokenize("NOT abc AND nothing"); - - it = result.iterator(); - Assert.assertEquals(NOT, it.next().getToken()); - Assert.assertEquals(WORD, it.next().getToken()); - Assert.assertEquals(AND, it.next().getToken()); - Assert.assertEquals(WORD, it.next().getToken()); - - result = tokenizer.tokenize("abc AND andsomething"); - - it = result.iterator(); - Assert.assertEquals(WORD, it.next().getToken()); - Assert.assertEquals(AND, it.next().getToken()); - Assert.assertEquals(WORD, it.next().getToken()); - - TokenizerValidator.init("abc AND ANDsomething") - .addExpected(WORD, AND, WORD).validate(); - - TokenizerValidator.init("abc ANDsomething") - .addExpected(WORD, WORD).validate(); - - TokenizerValidator.init("abc ORsomething") - .addExpected(WORD, WORD).validate(); - - TokenizerValidator.init("abc OR orsomething") - .addExpected(WORD, OR, WORD).validate(); - - TokenizerValidator.init("abc OR ORsomething") - .addExpected(WORD, OR, WORD).validate(); + assertQuery("NOT abc AND nothing").resultsIn(NOT, WORD, AND, WORD); + assertQuery("abc AND andsomething").resultsIn(WORD, AND, WORD); + assertQuery("abc AND ANDsomething").resultsIn(WORD, AND, WORD); + assertQuery("abc ANDsomething").resultsIn(WORD, WORD); + assertQuery("abc ORsomething").resultsIn(WORD, WORD); + assertQuery("abc OR orsomething").resultsIn(WORD, OR, WORD); + assertQuery("abc OR ORsomething").resultsIn(WORD, OR, WORD); } @Ignore @Test public void unicodeInWords() throws Exception { // Ll, Lm, Lo, Lt, Lu, Nl - TokenizerValidator.init("abc OR Ll\u01E3Lm\u02B5Lo\u1BE4Lt\u01F2Lu\u03D3Nl\u216F") - .addExpected(WORD, OR, WORD).validate(); + assertQuery("abc OR Ll\u01E3Lm\u02B5Lo\u1BE4Lt\u01F2Lu\u03D3Nl\u216F") + .resultsIn(WORD, OR, WORD); } /** @@ -369,8 +249,7 @@ public class SearchTokenizerTest { */ @Test public void characterInPhrase() throws Exception { - TokenizerValidator.init("\"123\" OR \"ALPHA-._~\"") - .addExpected(PHRASE, OR, PHRASE).validate(); + assertQuery("\"123\" OR \"ALPHA-._~\"").resultsIn(PHRASE, OR, PHRASE); } @Test @@ -395,7 +274,7 @@ public class SearchTokenizerTest { validate("abc def ghi"); // mixed not - TokenizerValidator.init(" abc def AND ghi").validate(WORD, WORD, AND, WORD); + assertQuery(" abc def AND ghi").resultsIn(WORD, WORD, AND, WORD); validate("NOT abc NOT def OR NOT ghi", NOT, WORD, NOT, WORD, OR, NOT, WORD); validate(" abc def NOT ghi", WORD, WORD, NOT, WORD); @@ -411,48 +290,60 @@ public class SearchTokenizerTest { @Test public void tokenizeInvalid() throws SearchTokenizerException { // - TokenizerValidator.init("( abc AND) OR something").validate(SearchTokenizerException.class); + assertQuery("( abc AND) OR something").resultsIn(SearchTokenizerException.class); - TokenizerValidator.init("\"phrase\"word").validate(SearchTokenizerException.class); - TokenizerValidator.init("\"p\"w").validate(SearchTokenizerException.class); - TokenizerValidator.init("\"\"").validate(SearchTokenizerException.class); + assertQuery("\"phrase\"word").resultsIn(SearchTokenizerException.class); + assertQuery("\"p\"w").resultsIn(SearchTokenizerException.class); + assertQuery("\"\"").resultsIn(SearchTokenizerException.class); } @Test public void tokenizeInvalidQueryForParser() throws SearchTokenizerException { - TokenizerValidator.init("AND").validate(AND); - TokenizerValidator.init("OR").validate(OR); - TokenizerValidator.init("NOT").validate(NOT); - TokenizerValidator.init("NOT AND").validate(NOT, AND); - TokenizerValidator.init("NOT OR").validate(NOT, OR); - TokenizerValidator.init("NOT NOT").validate(NOT, NOT); - TokenizerValidator.init("abc AND OR something").validate(WORD, AND, OR, WORD); - TokenizerValidator.init("abc AND \"something\" )").validate(WORD, AND, PHRASE, CLOSE); + assertQuery("AND").resultsIn(AND); + assertQuery("OR").resultsIn(OR); + assertQuery("NOT").resultsIn(NOT); + assertQuery("NOT AND").resultsIn(NOT, AND); + assertQuery("NOT OR").resultsIn(NOT, OR); + assertQuery("NOT NOT").resultsIn(NOT, NOT); + assertQuery("abc AND OR something").resultsIn(WORD, AND, OR, WORD); + assertQuery("abc AND \"something\" )").resultsIn(WORD, AND, PHRASE, CLOSE); } public void validate(String query) throws SearchTokenizerException { - new TokenizerValidator(query).validate(); + new Validator(query); + } + + public Validator assertQuery(String query) throws SearchTokenizerException { + return new Validator(query); } public void validate(String query, SearchQueryToken.Token ... tokens) throws SearchTokenizerException { - TokenizerValidator sv = new TokenizerValidator(query); + Validator sv = new Validator(query); for (SearchQueryToken.Token token : tokens) { sv.addExpected(token); } sv.validate(); } - private static class TokenizerValidator { + private static class Validator { private List validations = new ArrayList(); private boolean log; private final String searchQuery; - public void validate(SearchQueryToken.Token... tokens) throws SearchTokenizerException { + public void resultsIn(SearchQueryToken.Token... tokens) throws SearchTokenizerException { addExpected(tokens); validate(); } - - private class Tuple { + public void resultsIn(Tuple... tuple) throws SearchTokenizerException { + for (Tuple t : tuple) { + addExpected(t.token, t.literal); + } + validate(); + } + public static Tuple tuple(SearchQueryToken.Token token, String literal) { + return new Tuple(token, literal); + } + private static class Tuple { final SearchQueryToken.Token token; final String literal; public Tuple(SearchQueryToken.Token token, String literal) { @@ -464,30 +355,30 @@ public class SearchTokenizerTest { } } - private TokenizerValidator(String searchQuery) { + private Validator(String searchQuery) { this.searchQuery = searchQuery; } - private static TokenizerValidator init(String searchQuery) { - return new TokenizerValidator(searchQuery); + private static Validator init(String searchQuery) { + return new Validator(searchQuery); } @SuppressWarnings("unused") - private TokenizerValidator enableLogging() { + private Validator enableLogging() { log = true; return this; } - private TokenizerValidator addExpected(SearchQueryToken.Token token, String literal) { + private Validator addExpected(SearchQueryToken.Token token, String literal) { validations.add(new Tuple(token, literal)); return this; } - private TokenizerValidator addExpected(SearchQueryToken.Token ... token) { + private Validator addExpected(SearchQueryToken.Token ... token) { for (SearchQueryToken.Token t : token) { validations.add(new Tuple(t)); } return this; } - private void validate(Class exception) throws SearchTokenizerException { + private void resultsIn(Class exception) throws SearchTokenizerException { try { new SearchTokenizer().tokenize(searchQuery); } catch (Exception e) { @@ -497,6 +388,17 @@ public class SearchTokenizerTest { Assert.fail("Expected exception " + exception.getClass().getSimpleName() + " was not thrown."); } + private void resultsIn(SearchTokenizerException.MessageKey key) + throws SearchTokenizerException { + try { + init(searchQuery).validate(); + } catch (SearchTokenizerException e) { + Assert.assertEquals("SearchTokenizerException with unexpected message was thrown.", key, e.getMessageKey()); + return; + } + Assert.fail("No SearchTokenizerException was not thrown."); + } + private void validate() throws SearchTokenizerException { SearchTokenizer tokenizer = new SearchTokenizer(); List result = tokenizer.tokenize(searchQuery);