jackrabbit-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From thom...@apache.org
Subject svn commit: r1081847 - in /jackrabbit/sandbox/jackrabbit-j3/src: main/java/org/apache/jackrabbit/j3/json/ test/java/org/apache/jackrabbit/j3/json/
Date Tue, 15 Mar 2011 16:34:23 GMT
Author: thomasm
Date: Tue Mar 15 16:34:22 2011
New Revision: 1081847

URL: http://svn.apache.org/viewvc?rev=1081847&view=rev
Log:
Json/Jsop Tokenizer

Added:
    jackrabbit/sandbox/jackrabbit-j3/src/main/java/org/apache/jackrabbit/j3/json/
    jackrabbit/sandbox/jackrabbit-j3/src/main/java/org/apache/jackrabbit/j3/json/JsopTokenizer.java
    jackrabbit/sandbox/jackrabbit-j3/src/test/java/org/apache/jackrabbit/j3/json/
    jackrabbit/sandbox/jackrabbit-j3/src/test/java/org/apache/jackrabbit/j3/json/TestAll.java
    jackrabbit/sandbox/jackrabbit-j3/src/test/java/org/apache/jackrabbit/j3/json/TestTokenizer.java

Added: jackrabbit/sandbox/jackrabbit-j3/src/main/java/org/apache/jackrabbit/j3/json/JsopTokenizer.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-j3/src/main/java/org/apache/jackrabbit/j3/json/JsopTokenizer.java?rev=1081847&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-j3/src/main/java/org/apache/jackrabbit/j3/json/JsopTokenizer.java
(added)
+++ jackrabbit/sandbox/jackrabbit-j3/src/main/java/org/apache/jackrabbit/j3/json/JsopTokenizer.java
Tue Mar 15 16:34:22 2011
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.j3.json;
+
+/**
+ * A tokenizer for Json and Jsop strings.
+ */
+public class JsopTokenizer {
+
+    // "- " is "patch remove", without space is a negative number
+    // please note .0 and -.1 are not valid numbers
+
+    final static int END = 0;
+    final static int STRING = 1, NUMBER = 2, TRUE = 3, FALSE = 4, NULL = 5, ERROR = 6;
+
+    private final String jsop;
+    private int pos;
+    private int tokenType;
+    private String currentToken;
+
+    public JsopTokenizer(String json) {
+        this.jsop = json;
+    }
+
+    public String toString() {
+        return jsop;
+    }
+
+    /**
+     * Get the token type of the last token.
+     *
+     * @return the token type
+     */
+    public int getTokenType() {
+        return tokenType;
+    }
+
+    /**
+     * Get the last token value if the the token type was STRING or NUMBER. For
+     * STRING, the text is decoded; for NUMBER, it is returned as parsed. In all
+     * other cases the result is undefined.
+     *
+     * @return the token
+     */
+    public String getToken() {
+        return currentToken;
+    }
+
+    /**
+     * Read a token and return the token type.
+     *
+     * @return the token type
+     */
+    public int read() {
+        try {
+            return readToken();
+        } catch (StringIndexOutOfBoundsException e) {
+            tokenType = ERROR;
+            throw getFormatException(jsop, pos);
+        }
+    }
+
+    private int readToken() {
+        char ch;
+        while (true) {
+            if (pos >= jsop.length()) {
+                return tokenType = END;
+            }
+            ch = jsop.charAt(pos);
+            if (ch > ' ') {
+                break;
+            }
+            pos++;
+        }
+        int start = pos++;
+        switch (ch) {
+        case '\"': {
+            boolean escaped = false;
+            while (true) {
+                ch = jsop.charAt(pos++);
+                if (ch == '\"') {
+                    break;
+                } else if (ch == '\\') {
+                    escaped = true;
+                    pos++;
+                }
+            }
+            String s = jsop.substring(start + 1, pos - 1);
+            if (escaped) {
+                currentToken = jsonDecode(s);
+            } else {
+                currentToken = s;
+            }
+            tokenType = STRING;
+            break;
+        }
+        case '{':
+        case '}':
+        case '[':
+        case ']':
+        case '+':
+        case ':':
+        case ',':
+            tokenType = ch;
+            break;
+        case '-':
+            ch = jsop.charAt(pos);
+            if (ch < '0' || ch > '9') {
+                // lookahead
+                tokenType = '-';
+                break;
+            }
+            // else fall though
+        default:
+            if (ch >= '0' && ch <= '9') {
+                tokenType = NUMBER;
+                while (pos < jsop.length()) {
+                    ch = jsop.charAt(pos);
+                    if (ch < '0' || ch > '9') {
+                        break;
+                    }
+                    pos++;
+                }
+                if (ch == '.') {
+                    pos++;
+                    while (pos < jsop.length()) {
+                        ch = jsop.charAt(pos);
+                        if (ch < '0' || ch > '9') {
+                            break;
+                        }
+                        pos++;
+                    }
+                }
+                if (ch == 'e' || ch == 'E') {
+                    ch = jsop.charAt(++pos);
+                    if (ch == '+' || ch == '-') {
+                        ch = jsop.charAt(++pos);
+                    }
+                    while (pos < jsop.length()) {
+                        ch = jsop.charAt(pos);
+                        if (ch < '0' || ch > '9') {
+                            break;
+                        }
+                        pos++;
+                    }
+                }
+                currentToken = jsop.substring(start, pos);
+            } else if (ch >= 'a' && ch <= 'z') {
+                while (pos < jsop.length()) {
+                    ch = jsop.charAt(pos);
+                    if (ch < 'a' || ch > 'z') {
+                        break;
+                    }
+                    pos++;
+                }
+                String s = jsop.substring(start, pos);
+                if ("null".equals(s)) {
+                    tokenType = NULL;
+                } else if ("true".equals(s)) {
+                    tokenType = TRUE;
+                } else if ("false".equals(s)) {
+                    tokenType = FALSE;
+                } else {
+                    tokenType = ERROR;
+                    throw getFormatException(jsop, pos);
+                }
+                break;
+            } else {
+                tokenType = ERROR;
+                throw getFormatException(jsop, pos);
+            }
+        }
+        return tokenType;
+    }
+
+    /**
+     * Decode a Json string.
+     *
+     * @param s the encoded string
+     * @return the string
+     */
+    public static String jsonDecode(String s) {
+        int length = s.length();
+        StringBuilder buff = new StringBuilder(length);
+        for (int i = 0; i < length; i++) {
+            char c = s.charAt(i);
+            if (c == '\\') {
+                if (i + 1 >= s.length()) {
+                    throw getFormatException(s, i);
+                }
+                c = s.charAt(++i);
+                switch (c) {
+                case '"':
+                    buff.append('"');
+                    break;
+                case '\\':
+                    buff.append('\\');
+                    break;
+                case '/':
+                    buff.append('/');
+                    break;
+                case 'b':
+                    buff.append('\b');
+                    break;
+                case 'f':
+                    buff.append('\f');
+                    break;
+                case 'n':
+                    buff.append('\n');
+                    break;
+                case 'r':
+                    buff.append('\r');
+                    break;
+                case 't':
+                    buff.append('\t');
+                    break;
+                case 'u': {
+                    try {
+                        c = (char) (Integer.parseInt(s.substring(i + 1, i + 5), 16));
+                    } catch (NumberFormatException e) {
+                        throw getFormatException(s, i);
+                    }
+                    i += 4;
+                    buff.append(c);
+                    break;
+                }
+                default:
+                    throw getFormatException(s, i);
+                }
+            } else {
+                buff.append(c);
+            }
+        }
+        return buff.toString();
+    }
+
+    /**
+     * Convert a string to a Json literal using the correct escape sequences.
+     * The literal is enclosed in double quotes. Characters outside the range
+     * 32..127 are encoded (backslash u xxxx). The forward slash (solidus) is
+     * not escaped.
+     *
+     * @param s the text to convert
+     * @return the Json representation (including double quotes)
+     */
+    public static String jsonEncode(String s) {
+        int length = s.length();
+        StringBuilder buff = new StringBuilder(length + 2);
+        buff.append('\"');
+        for (int i = 0; i < length; i++) {
+            char c = s.charAt(i);
+            switch (c) {
+            case '"':
+                // quotation mark
+                buff.append("\\\"");
+                break;
+            case '\\':
+                // backslash
+                buff.append("\\\\");
+                break;
+            case '\b':
+                // backspace
+                buff.append("\\b");
+                break;
+            case '\f':
+                // formfeed
+                buff.append("\\f");
+                break;
+            case '\n':
+                // newline
+                buff.append("\\n");
+                break;
+            case '\r':
+                // carriage return
+                buff.append("\\r");
+                break;
+            case '\t':
+                // horizontal tab
+                buff.append("\\t");
+                break;
+            default:
+                int ch = c;
+                if (ch >= ' ' && ch <= 127) {
+                    buff.append(c);
+                } else {
+                    buff.append("\\u");
+                    // make sure it's four characters
+                    buff.append(Integer.toHexString(0x10000 | ch).substring(1));
+                }
+            }
+        }
+        buff.append('\"');
+        return buff.toString();
+    }
+
+    private static IllegalArgumentException getFormatException(String s, int i) {
+        return new IllegalArgumentException(addAsterisk(s, i));
+    }
+
+    /**
+     * Add an asterisk ('[*]') at the given position. This format is used to
+     * show where parsing failed in a statement.
+     *
+     * @param s the text
+     * @param index the position
+     * @return the text with asterisk
+     */
+    private static String addAsterisk(String s, int index) {
+        if (s != null && index < s.length()) {
+            s = s.substring(0, index) + "[*]" + s.substring(index);
+        }
+        return s;
+    }
+
+}

Added: jackrabbit/sandbox/jackrabbit-j3/src/test/java/org/apache/jackrabbit/j3/json/TestAll.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-j3/src/test/java/org/apache/jackrabbit/j3/json/TestAll.java?rev=1081847&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-j3/src/test/java/org/apache/jackrabbit/j3/json/TestAll.java
(added)
+++ jackrabbit/sandbox/jackrabbit-j3/src/test/java/org/apache/jackrabbit/j3/json/TestAll.java
Tue Mar 15 16:34:22 2011
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.j3.json;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * Runs all tests.
+ */
+public class TestAll extends TestCase {
+
+    /**
+     * Returns a <code>Test</code> suite that executes all tests inside this
+     * package.
+     *
+     * @return a <code>Test</code> suite that executes all tests inside this
+     *         package.
+     */
+    public static Test suite() {
+        TestSuite suite = new TestSuite("org.apache.jackrabbit.j3.json");
+
+        suite.addTestSuite(TestTokenizer.class);
+
+        return suite;
+    }
+}

Added: jackrabbit/sandbox/jackrabbit-j3/src/test/java/org/apache/jackrabbit/j3/json/TestTokenizer.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-j3/src/test/java/org/apache/jackrabbit/j3/json/TestTokenizer.java?rev=1081847&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-j3/src/test/java/org/apache/jackrabbit/j3/json/TestTokenizer.java
(added)
+++ jackrabbit/sandbox/jackrabbit-j3/src/test/java/org/apache/jackrabbit/j3/json/TestTokenizer.java
Tue Mar 15 16:34:22 2011
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.j3.json;
+
+import junit.framework.TestCase;
+
+/**
+ * Test the Jsop tokenizer.
+ */
+public class TestTokenizer extends TestCase {
+
+    public void test() {
+        try {
+            JsopTokenizer.jsonDecode("test\\");
+            fail();
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+        try {
+            JsopTokenizer.jsonDecode("wrong\\uxxxx");
+            fail();
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+        try {
+            JsopTokenizer.jsonDecode("wrong\\m");
+            fail();
+        } catch (IllegalArgumentException e) {
+            // ok
+        }
+        test("/* error */", "\"\\");
+        test("/* error */1", ".1");
+        assertEquals("x", new JsopTokenizer("x").toString());
+        test("/* error */", "true" + "true");
+        test("/* error */", "truer");
+        test("/* error */", "falsehood");
+        test("/* error */", "nil");
+        test("/* error */", "\"invalid");
+        test("- \"test/test\"", "-\"test\\/test\"");
+        test(" {\n\"x\": 1, \"y\": 2\n}\n", "{\"x\":1, \"y\":2}");
+        test(" [\ntrue, false, null\n]\n", "[true, false, null]");
+        test("\"-\\\\-\\\"-\\b-\\f-\\n-\\r-\\t-\\u1234\"", "\"-\\\\-\\\"-\\b-\\f-\\n-\\r-\\t-\\u1234\"");
+        test("\"-\\b-\\f-\\n-\\r-\\t\"", "\"-\b-\f-\n-\r-\t\"");
+        test(" [\n0, 12, -1, 0.1, -0.1, -2.3e1, 1e+1, 1.e-20\n]\n", "[0,12,-1,0.1,-0.1,-2.3e1,1e+1,1.e-20]");
+        test("\"Hello\"", "\"Hello\"");
+        test(" [\n\n]\n", "[]");
+        test(" {\n\n}\n", "{}");
+    }
+
+    static void test(String expected, String json) {
+        String j2 = prettyPrintWithErrors(json);
+        assertEquals(expected, j2);
+    }
+
+    static String prettyPrintWithErrors(String jsop) {
+        StringBuilder buff = new StringBuilder();
+        JsopTokenizer t = new JsopTokenizer(jsop);
+        while (true) {
+            try {
+                prettyPrint(buff, t);
+            } catch (IllegalArgumentException e) {
+                buff.append("/* error */");
+            }
+            if (t.getTokenType() == JsopTokenizer.END) {
+                return buff.toString();
+            }
+        }
+    }
+
+    static String prettyPrint(String jsop) {
+        StringBuilder buff = new StringBuilder();
+        JsopTokenizer t = new JsopTokenizer(jsop);
+        return prettyPrint(buff, t);
+    }
+
+    static String prettyPrint(StringBuilder buff, JsopTokenizer t) {
+        while (true) {
+            switch (t.read()) {
+            case JsopTokenizer.END:
+                return buff.toString();
+            case JsopTokenizer.STRING:
+                buff.append(JsopTokenizer.jsonEncode(t.getToken()));
+                break;
+            case JsopTokenizer.NUMBER:
+                buff.append(t.getToken());
+                break;
+            case JsopTokenizer.TRUE:
+                buff.append("true");
+                break;
+            case JsopTokenizer.FALSE:
+                buff.append("false");
+                break;
+            case JsopTokenizer.NULL:
+                buff.append("null");
+                break;
+            case '{':
+                buff.append(" {\n");
+                break;
+            case '}':
+                buff.append("\n}\n");
+                break;
+            case '[':
+                buff.append(" [\n");
+                break;
+            case ']':
+                buff.append("\n]\n");
+                break;
+            case ',':
+                buff.append(", ");
+                break;
+            case ':':
+                buff.append(": ");
+                break;
+            case '+':
+                buff.append("+ ");
+                break;
+            case '-':
+                buff.append("- ");
+                break;
+            case '^':
+                buff.append("^ ");
+                break;
+            default:
+                throw new IllegalArgumentException("token type: " + t.getTokenType());
+            }
+        }
+    }
+
+}



Mime
View raw message