pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1781129 - in /pig/trunk: ./ ivy/ src/org/apache/pig/ src/org/apache/pig/tools/grunt/ src/org/apache/pig/tools/parameters/ src/org/apache/pig/tools/pigscript/parser/ test/org/apache/pig/test/
Date Tue, 31 Jan 2017 17:36:45 GMT
Author: daijy
Date: Tue Jan 31 17:36:45 2017
New Revision: 1781129

URL: http://svn.apache.org/viewvc?rev=1781129&view=rev
Log:
PIG-3851: Upgrade jline to 2.11

Added:
    pig/trunk/src/org/apache/pig/tools/grunt/ConsoleReaderInputStream.java
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/ivy/libraries.properties
    pig/trunk/src/org/apache/pig/Main.java
    pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java
    pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
    pig/trunk/src/org/apache/pig/tools/grunt/PigCompletor.java
    pig/trunk/src/org/apache/pig/tools/grunt/PigCompletorAliases.java
    pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
    pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
    pig/trunk/test/org/apache/pig/test/TestGrunt.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jan 31 17:36:45 2017
@@ -34,6 +34,8 @@ PIG-5067: Revisit union on numeric type
  
 IMPROVEMENTS
 
+PIG-3851: Upgrade jline to 2.11 (daijy)
+
 PIG-4963: Add a Bloom join (rohini)
 
 PIG-3938: Add LoadCaster to EvalFunc (knoguchi)

Modified: pig/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/trunk/ivy/libraries.properties?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/ivy/libraries.properties (original)
+++ pig/trunk/ivy/libraries.properties Tue Jan 31 17:36:45 2017
@@ -56,7 +56,7 @@ jdeb.version=0.8
 jdiff.version=1.0.9
 jettison.version=1.3.4
 jetty.version=6.1.26
-jline.version=1.0
+jline.version=2.11
 joda-time.version=2.9.3
 jopt.version=4.1
 json-simple.version=1.1

Modified: pig/trunk/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Tue Jan 31 17:36:45 2017
@@ -27,7 +27,6 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
 import java.io.Reader;
 import java.io.StringReader;
 import java.net.URL;
@@ -45,9 +44,8 @@ import java.util.jar.Attributes;
 import java.util.jar.JarFile;
 import java.util.jar.Manifest;
 
-import jline.ConsoleReader;
-import jline.ConsoleReaderInputStream;
-import jline.History;
+import jline.console.ConsoleReader;
+import jline.console.history.FileHistory;
 
 import org.antlr.runtime.RecognitionException;
 import org.apache.commons.logging.Log;
@@ -77,6 +75,7 @@ import org.apache.pig.parser.DryRunGrunt
 import org.apache.pig.scripting.ScriptEngine;
 import org.apache.pig.scripting.ScriptEngine.SupportedScriptLang;
 import org.apache.pig.tools.cmdline.CmdLineParser;
+import org.apache.pig.tools.grunt.ConsoleReaderInputStream;
 import org.apache.pig.tools.grunt.Grunt;
 import org.apache.pig.tools.pigstats.PigProgressNotificationListener;
 import org.apache.pig.tools.pigstats.PigStats;
@@ -551,12 +550,13 @@ public class Main {
                 }
                 // Interactive
                 mode = ExecMode.SHELL;
-              //Reader is created by first loading "pig.load.default.statements" or .pigbootup
file if available
-                ConsoleReader reader = new ConsoleReader(Utils.getCompositeStream(System.in,
properties), new OutputStreamWriter(System.out));
-                reader.setDefaultPrompt("grunt> ");
+                //Reader is created by first loading "pig.load.default.statements" or .pigbootup
file if available
+                ConsoleReader reader = new ConsoleReader(Utils.getCompositeStream(System.in,
properties), System.out);
+                reader.setExpandEvents(false);
+                reader.setPrompt("grunt> ");
                 final String HISTORYFILE = ".pig_history";
                 String historyFile = System.getProperty("user.home") + File.separator  +
HISTORYFILE;
-                reader.setHistory(new History(new File(historyFile)));
+                reader.setHistory(new FileHistory(new File(historyFile)));
                 ConsoleReaderInputStream inputStream = new ConsoleReaderInputStream(reader);
                 grunt = new Grunt(new BufferedReader(new InputStreamReader(inputStream)),
pigContext);
                 grunt.setConsoleReader(reader);

Added: pig/trunk/src/org/apache/pig/tools/grunt/ConsoleReaderInputStream.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/ConsoleReaderInputStream.java?rev=1781129&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/ConsoleReaderInputStream.java (added)
+++ pig/trunk/src/org/apache/pig/tools/grunt/ConsoleReaderInputStream.java Tue Jan 31 17:36:45
2017
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.tools.grunt;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+import java.util.Enumeration;
+
+import jline.console.ConsoleReader;
+
+/** Borrowed from jline.console.internal.ConsoleReaderInputStream. However,
+ *  we cannot use ConsoleReaderInputStream directly since:
+ *  1. ConsoleReaderInputStream is not public
+ *  2. ConsoleReaderInputStream has a bug which does not deal with UTF-8 correctly
+ */
+public class ConsoleReaderInputStream extends SequenceInputStream {
+    private static InputStream systemIn = System.in;
+
+    public static void setIn() throws IOException {
+        setIn(new ConsoleReader());
+    }
+
+    public static void setIn(final ConsoleReader reader) {
+        System.setIn(new ConsoleReaderInputStream(reader));
+    }
+
+    /**
+     * Restore the original {@link System#in} input stream.
+     */
+    public static void restoreIn() {
+        System.setIn(systemIn);
+    }
+
+    public ConsoleReaderInputStream(final ConsoleReader reader) {
+        super(new ConsoleEnumeration(reader));
+    }
+
+    private static class ConsoleEnumeration implements Enumeration {
+        private final ConsoleReader reader;
+        private ConsoleLineInputStream next = null;
+        private ConsoleLineInputStream prev = null;
+
+        public ConsoleEnumeration(final ConsoleReader reader) {
+            this.reader = reader;
+        }
+
+        public Object nextElement() {
+            if (next != null) {
+                InputStream n = next;
+                prev = next;
+                next = null;
+
+                return n;
+            }
+
+            return new ConsoleLineInputStream(reader);
+        }
+
+        public boolean hasMoreElements() {
+            // the last line was null
+            if ((prev != null) && (prev.wasNull == true)) {
+                return false;
+            }
+
+            if (next == null) {
+                next = (ConsoleLineInputStream) nextElement();
+            }
+
+            return next != null;
+        }
+    }
+
+    private static class ConsoleLineInputStream extends InputStream {
+        private final ConsoleReader reader;
+        private byte[] buffer = null;
+        private int index = 0;
+        private boolean eol = false;
+        protected boolean wasNull = false;
+
+        public ConsoleLineInputStream(final ConsoleReader reader) {
+            this.reader = reader;
+        }
+
+        public int read() throws IOException {
+            if (eol) {
+                return -1;
+            }
+
+            if (buffer == null) {
+                buffer = reader.readLine().getBytes();
+            }
+
+            if (buffer == null) {
+                wasNull = true;
+                return -1;
+            }
+
+            if (index >= buffer.length) {
+                eol = true;
+                return '\n'; // lines are ended with a newline
+            }
+
+            return buffer[index++];
+        }
+    }
+}
\ No newline at end of file

Modified: pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java Tue Jan 31 17:36:45 2017
@@ -20,7 +20,7 @@ package org.apache.pig.tools.grunt;
 import java.io.BufferedReader;
 import java.util.ArrayList;
 
-import jline.ConsoleReader;
+import jline.console.ConsoleReader;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -52,8 +52,8 @@ public class Grunt
 
     public void setConsoleReader(ConsoleReader c)
     {
-        c.addCompletor(new PigCompletorAliases(pig));
-        c.addCompletor(new PigCompletor());
+        c.addCompleter(new PigCompletorAliases(pig));
+        c.addCompleter(new PigCompletor());
         parser.setConsoleReader(c);
     }
 

Modified: pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Tue Jan 31 17:36:45 2017
@@ -26,7 +26,6 @@ import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
 import java.io.PrintStream;
 import java.io.Reader;
 import java.io.StringReader;
@@ -42,8 +41,7 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 
-import jline.ConsoleReader;
-import jline.ConsoleReaderInputStream;
+import jline.console.ConsoleReader;
 
 import org.apache.commons.io.output.NullOutputStream;
 import org.apache.commons.logging.Log;
@@ -264,7 +262,7 @@ public class GruntParser extends PigScri
     public void prompt()
     {
         if (mInteractive) {
-            mConsoleReader.setDefaultPrompt("grunt> ");
+            mConsoleReader.setPrompt("grunt> ");
         }
     }
 
@@ -533,7 +531,7 @@ public class GruntParser extends PigScri
                 cmds = cmds.replaceAll("\t","    ");
 
                 reader = new ConsoleReader(new ByteArrayInputStream(cmds.getBytes()),
-                                           new OutputStreamWriter(System.out));
+                                           System.out);
                 reader.setHistory(mConsoleReader.getHistory());
                 InputStream in = new ConsoleReaderInputStream(reader);
                 inputReader = new BufferedReader(new InputStreamReader(in));

Modified: pig/trunk/src/org/apache/pig/tools/grunt/PigCompletor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/PigCompletor.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/PigCompletor.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/PigCompletor.java Tue Jan 31 17:36:45 2017
@@ -33,9 +33,9 @@ import java.util.TreeSet;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
-import jline.Completor;
+import jline.console.completer.Completer;
 
-public class PigCompletor implements Completor {
+public class PigCompletor implements Completer {
     private final Log log = LogFactory.getLog(getClass());
     Set<String> candidates;
     static final String AUTOCOMPLETE_FILENAME = "autocomplete";

Modified: pig/trunk/src/org/apache/pig/tools/grunt/PigCompletorAliases.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/PigCompletorAliases.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/PigCompletorAliases.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/PigCompletorAliases.java Tue Jan 31 17:36:45
2017
@@ -26,12 +26,11 @@ import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.apache.pig.PigServer;
 
-import jline.Completor;
+import jline.console.completer.Completer;
 
-public class PigCompletorAliases implements Completor {
+public class PigCompletorAliases implements Completer {
     private final Log log = LogFactory.getLog(getClass());
     Set<String> keywords;
     PigServer pig;

Modified: pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original)
+++ pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java Tue Jan 31 17:36:45
2017
@@ -42,7 +42,6 @@ import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.validator.BlackAndWhitelistFilter;
 import org.apache.pig.validator.PigCommandFilter;
-import org.python.google.common.base.Preconditions;
 
 public class PreprocessorContext {
 

Modified: pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original)
+++ pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Tue Jan 31 17:36:45
2017
@@ -37,7 +37,7 @@ import java.util.List;
 import java.util.ArrayList;
 import org.apache.pig.impl.util.StringUtils;
 
-import jline.ConsoleReader;
+import jline.console.ConsoleReader;
 
 public abstract class PigScriptParser
 {
@@ -218,7 +218,7 @@ TOKEN_MGR_DECLS : {
 		{
 			/*System.err.print(">> ");
 			System.err.flush();*/
-		    consoleReader.setDefaultPrompt(">> ");
+		    consoleReader.setPrompt(">> ");
 		}
 	}
 

Modified: pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestGrunt.java Tue Jan 31 17:36:45 2017
@@ -28,6 +28,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.FileWriter;
+import java.io.FilenameFilter;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
@@ -1555,4 +1556,20 @@ public class TestGrunt {
         }
         assertTrue(found);
     }
+
+    @Test
+    public void testGruntUtf8() throws Throwable {
+        String command = "mkdir 测试\n" +
+                "quit\n";
+        System.setProperty("jline.WindowsTerminal.directConsole", "false");
+        System.setIn(new ByteArrayInputStream(command.getBytes()));
+        org.apache.pig.PigRunner.run(new String[] {"-x", "local"}, null);
+        File[] partFiles = new File(".").listFiles(new FilenameFilter() {
+            public boolean accept(File dir, String name) { 
+            return name.equals("测试");
+        }
+        });
+        assertEquals(partFiles.length, 1);
+        new File("测试").delete();
+    }
 }



Mime
View raw message