db-derby-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fuzzylo...@apache.org
Subject svn commit: r394770 - in /db/derby/code/trunk/java: testing/org/apache/derbyTesting/functionTests/harness/ tools/org/apache/derby/iapi/tools/i18n/ tools/org/apache/derby/impl/tools/ij/
Date Mon, 17 Apr 2006 20:30:25 GMT
Author: fuzzylogic
Date: Mon Apr 17 13:30:21 2006
New Revision: 394770

URL: http://svn.apache.org/viewcvs?rev=394770&view=rev
Log:
DERBY-658: Improve the test harness' ability to run on platforms with non-ASCII
native encoding.

Committed for Myrna Van Lunteren <m.v.lunteren@gmail.com>

Modified:
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/CopySuppFiles.java
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/FileCompare.java
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/RunList.java
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/RunTest.java
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/Sed.java
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/SkipTest.java
    db/derby/code/trunk/java/tools/org/apache/derby/iapi/tools/i18n/LocalizedResource.java
    db/derby/code/trunk/java/tools/org/apache/derby/impl/tools/ij/utilMain.java

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/CopySuppFiles.java
URL: http://svn.apache.org/viewcvs/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/CopySuppFiles.java?rev=394770&r1=394769&r2=394770&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/CopySuppFiles.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/CopySuppFiles.java
Mon Apr 17 13:30:21 2006
@@ -66,17 +66,31 @@
 		    fw.close();
 		    suppFile = new File(f.getCanonicalPath());
 		}
-
-
-    			FileOutputStream fos = new FileOutputStream(suppFile);
-                byte[] data = new byte[4096];
-                int len;
-    			while ((len = is.read(data)) != -1)
-    			{
-    			    fos.write(data, 0, len);
-    			}
-    			fos.close();
-			}
+                // need to make a guess so we copy text files to local encoding
+                // on non-ascii systems...
+		        if ((fileName.indexOf("sql") > 0) || (fileName.indexOf("txt") > 0) || (fileName.indexOf(".view")
> 0) || (fileName.indexOf(".policy") > 0))
+                {
+                    BufferedReader inFile = new BufferedReader(new InputStreamReader(is,
"UTF-8"));
+                    PrintWriter pw = new PrintWriter
+                       ( new BufferedWriter(new FileWriter(suppFile), 10000), true );
+                    int c;
+                    while ((c = inFile.read()) != -1)
+                        pw.write(c);
+                    pw.flush();
+                    pw.close();
+                }
+                else
+                {
+                    FileOutputStream fos = new FileOutputStream(suppFile);
+                    byte[] data = new byte[4096];
+                    int len;
+                    while ((len = is.read(data)) != -1)
+                    {
+                        fos.write(data, 0, len);
+                    }
+                    fos.close();
+                }
+    		}
         }
 	}
 }

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/FileCompare.java
URL: http://svn.apache.org/viewcvs/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/FileCompare.java?rev=394770&r1=394769&r2=394770&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/FileCompare.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/FileCompare.java
Mon Apr 17 13:30:21 2006
@@ -199,7 +199,22 @@
 		    {
 		        System.out.println("SED Error: " + cfe.getMessage());
 		    }
-		}
+		}	
+        else 
+        {
+            // read in in fixed format, but write out relying on default encoding
+            File EncodedOutFile = new File(outDir, testBase + ".tmpmstr");
+            BufferedReader inFile = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+            
+            PrintWriter pw = new PrintWriter
+            ( new BufferedWriter(new FileWriter(EncodedOutFile), 10000), true );
+            int c;
+            while ((c = inFile.read()) != -1)
+                pw.write(c);
+            pw.flush();
+            pw.close();
+            is = new FileInputStream(EncodedOutFile);
+        }	
         
         // Define the input and output files
         outFile = new BufferedReader(new FileReader(outfile));

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/RunList.java
URL: http://svn.apache.org/viewcvs/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/RunList.java?rev=394770&r1=394769&r2=394770&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/RunList.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/RunList.java
Mon Apr 17 13:30:21 2006
@@ -227,7 +227,7 @@
             }
 
             // Create a BufferedReader to read the list of tests to run
-            runlistFile = new BufferedReader(new InputStreamReader(is));
+            runlistFile = new BufferedReader(new InputStreamReader(is, "UTF-8"));
             if (runlistFile == null)
             {
                 System.out.println("The suite runall file could not be read.");

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/RunTest.java
URL: http://svn.apache.org/viewcvs/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/RunTest.java?rev=394770&r1=394769&r2=394770&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/RunTest.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/RunTest.java
Mon Apr 17 13:30:21 2006
@@ -36,6 +36,7 @@
 import java.io.BufferedOutputStream;
 import java.io.BufferedWriter;
 import java.io.FileWriter;
+import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.io.PrintStream;
 import java.io.IOException;
@@ -103,6 +104,7 @@
 	static boolean upgradetest = false;
 	static boolean encryption = false; // requires jdk12ext plus encryptionProtocol
 	static boolean jdk12exttest = false; // requires jdk12ext
+    static boolean generateUTF8Out = false; // setting to create a utf8 encoded master file.
 	static String runningdir = ""; // where the tests are run and suppfiles placed
 	static String outputdir = ""; // user can specify as a property (optional)
 	static String canondir; // optional (to specify other than "master")
@@ -135,8 +137,9 @@
 	static String testOutName; // output name without path or extension (optional)
 	static String passFileName; // file listing passed tests
 	static String failFileName; // file listing failed tests
-	static String JCCOutName; //file name for JCC corrected master
-    static File passFile;
+    static String UTF8OutName; // file name for utf8 encoded out - not used for file comparison
+	static String tempMasterName; //file name for master, converted to local encoding, and for
network server, corrected master
+	static File passFile;
     static File failFile;
 	static String shutdownurl = "";
     static boolean useOutput; // use output or assume .tmp file is produced?
@@ -152,9 +155,10 @@
     static File runDir; // where test is run and where support files are expected
     static File canonDir; // allows setting master dir other than default
     static File tmpOutFile; // tmp output file (before sed)
-    static File JCCOutFile; // master file processed for JCC
+    static File tempMasterFile; // master file copied into local encoding - with networkserver,
also processed
     static File stdOutFile; // for tests with useoutput false
     static File finalOutFile; // final output file (after sed)
+    static File UTF8OutFile; // file name for out file copied into utf8 encoding
     static File appPropFile; // testname_app.properties or default
     static File clPropFile; // testname_derby.properties or default
     static File diffFile; // To indicate diffs
@@ -409,9 +413,10 @@
     		pwDiff.flush();
         }
 
+        generateUTF8OutFile(finalOutFile);
+        
 		// Cleanup files
 		doCleanup(javaVersion);
-
 	}
 
     private static void testRun(String propString, Properties sysProp)
@@ -643,7 +648,7 @@
 
             // Read the test file and copy it to the outDir
             // except for multi tests (for multi we just need to locate it)
-            BufferedReader in = new BufferedReader(new InputStreamReader(is));
+            BufferedReader in = new BufferedReader(new InputStreamReader(is, "UTF-8"));
             if (upgradetest)
 		
                 //these calls to getCanonicalPath catch IOExceptions as a workaround to
@@ -765,11 +770,10 @@
 
         // Create a .tmp file for doing sed later to create testBase.out
         tmpOutFile = new File(outDir, testOutName + ".tmp");
-		if (NetServer.isClientConnection(framework))
-		{
-			JCCOutName = testOutName+".tmpmstr";
-		}
-
+        // Always create a.tmpmstr copy of the master file in local encoding.
+        // With network server, this gets adjusted for displaywidth 
+        tempMasterName = testOutName+".tmpmstr";
+		UTF8OutName = testOutName+".utf8out";
 		// Define the .out file which will be created by massaging the tmp.out
 		finalOutFile = new File(outDir, testOutName + ".out");
 
@@ -801,18 +805,18 @@
         // Delete any old .out or .tmp files
         if (tmpOutFile.exists())
             status = tmpOutFile.delete();
-		if (NetServer.isClientConnection(framework))
-		{
-        	JCCOutFile = new File(outDir, JCCOutName);
-        	if (JCCOutFile.exists())
-            	status = JCCOutFile.delete();
-		}
-        if (finalOutFile.exists())
+        tempMasterFile = new File(outDir, tempMasterName);
+        if (tempMasterFile.exists())
+            status = tempMasterFile.delete();
+		if (finalOutFile.exists())
             status = finalOutFile.delete();
         if (diffFile.exists())
             status = diffFile.delete();
         if (stdOutFile.exists())
             status = stdOutFile.delete();
+        UTF8OutFile = new File (outDir, UTF8OutName);
+        if (UTF8OutFile.exists())
+            status = UTF8OutFile.delete();
 
         // Delete any old pass or fail files
         if (!isSuiteRun)
@@ -849,6 +853,10 @@
         if (hostName == null)
            hostName="localhost";
 		
+        String generateUTF8OutProp = sp.getProperty("generateUTF8Out");
+        if (generateUTF8OutProp != null && generateUTF8OutProp.equals("true"))
+        	generateUTF8Out = true;
+        
         // Some tests will not work with some frameworks,
         // so check suite exclude files for tests to be skipped
         String skipFile = framework + ".exclude";
@@ -1408,6 +1416,7 @@
 			{
 				ap.put("derby.ui.codeset",fileEnc);
 			}
+
 			if (verbose)
 				System.out.println("console.encoding:" + conEnc + 
 								   " file.encoding:" + fileEnc +
@@ -1820,8 +1829,8 @@
         //printWriter.close();
         //printWriter = null;
 
-        //Always cleanup the script files
-        if ( !(script == null) && (script.exists()) )
+        //Always cleanup the script files - except when keepfiles is true
+        if ( !(script == null) && (script.exists()) && (!keepfiles) )
         {
             status = script.delete();
             //System.out.println("Status was: " + status);
@@ -1850,13 +1859,12 @@
                 tmpOutFile = null;
             status = finalOutFile.delete();
             if (skiptest == false)
-                status = diffFile.delete();
-			// delete JCC filtered master file
-			if (NetServer.isClientConnection(framework))
-			{
-        		JCCOutFile = new File(outDir, JCCOutName);
-            	status = JCCOutFile.delete();
-			}
+                status = diffFile.delete();           
+            // delete the copied (and for Network Server, modified) master file 
+            tempMasterFile = new File(outDir, tempMasterName);
+            status = tempMasterFile.delete();
+            UTF8OutFile = new File(outDir, UTF8OutName);
+            status = UTF8OutFile.delete();
             if (deleteBaseDir)
             {
                 if (useCommonDB == false) 
@@ -1995,7 +2003,7 @@
     {
         PrintWriter tmpPw = new PrintWriter(bos);
         // reader for stderr
-        BufferedReader errReader = new BufferedReader(new InputStreamReader(is));
+        BufferedReader errReader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
         String s = null;
         int lines = 0;
         while ((s = errReader.readLine()) != null)
@@ -2133,18 +2141,23 @@
         Vector v = jvm.getCommandLine();
         if ( ij.startsWith("ij") )
         {
-            // as of cn1411-20030930, the system takes the default console encoding
-            // which in the US, on windows, is Cp437.
-            // Sun on the other hand, always forces a console encoding of 1252.
-            // To get the same result for ibm141 & jdk14*, we need to force 
-            // the console encoding to Cp1252 for ij tests.
-            // see beetle 5475.
-            v.addElement("-Dconsole.encoding=Cp1252" );
+            // As of cn1411-20030930 IBM jvm the system takes the default
+            // console encoding which in the US, on windows, is Cp437.
+            // Sun jvms, however, always force a console encoding of 1252.
+            // To get the same result for ibm141 & jdk14*, the harness needs to
+            // force the console encoding to Cp1252 for ij tests - unless 
+            // we're on non-ascii systems.
+            String isNotAscii = System.getProperty("platform.notASCII");
+            if ( (isNotAscii == null) || (isNotAscii.equals("false"))) 
+                v.addElement("-Dconsole.encoding=Cp1252" );
             v.addElement("org.apache.derby.tools." + ij);
             if (ij.equals("ij"))
             {
-                v.addElement("-fr");
-                v.addElement(scriptFileName);
+                //TODO: is there a setting/property we could check after which
+            	// we can use v.addElement("-fr"); (read from the classpath?)
+                // then we can also use v.addElement(scriptFile);
+            	v.addElement("-f");
+                v.addElement(outDir.toString() + File.separatorChar + scriptFileName);
             }
             v.addElement("-p");
             v.addElement(propString);
@@ -2568,6 +2581,29 @@
     	}
         
         return installedSecurityManager;
+    }
+    
+    // copy the .out file in utf8 format. 
+    // This can then be used as a new master in the source.
+    // It is assumed that if one runs with this property, keepfiles should be true.
+    private static void generateUTF8OutFile(File FinalOutFile) throws IOException
+    {
+        if (generateUTF8Out) 
+        {
+            keepfiles=true;
+        	File UTF8OutFile = new File(UTF8OutName);
+        	
+        	// start reading the .out file back in, using default encoding
+        	BufferedReader inFile = new BufferedReader(new FileReader(FinalOutFile));
+        	FileOutputStream fos = new FileOutputStream(UTF8OutFile);
+        	BufferedWriter bw = new BufferedWriter (new OutputStreamWriter(fos, "UTF-8")); 

+        	int c;
+        	while ((c = inFile.read()) != -1)
+        		bw.write(c);
+        	bw.flush();
+        	bw.close();
+        	fos.close();     
+        }
     }
 
 }

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/Sed.java
URL: http://svn.apache.org/viewcvs/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/Sed.java?rev=394770&r1=394769&r2=394770&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/Sed.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/Sed.java
Mon Apr 17 13:30:21 2006
@@ -357,7 +357,7 @@
         if (is == null)
             inFile = new BufferedReader(new FileReader(srcFile));
         else
-            inFile = new BufferedReader(new InputStreamReader(is));
+            inFile = new BufferedReader(new InputStreamReader(is, "UTF-8"));
         outFile = new PrintWriter
         ( new BufferedWriter(new FileWriter(dstFile), 10000), true );
 

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/SkipTest.java
URL: http://svn.apache.org/viewcvs/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/SkipTest.java?rev=394770&r1=394769&r2=394770&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/SkipTest.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/harness/SkipTest.java
Mon Apr 17 13:30:21 2006
@@ -53,7 +53,7 @@
         }
         
         // Create a BufferedReader to read the list of tests to skip
-        BufferedReader listFile = new BufferedReader(new InputStreamReader(is));
+        BufferedReader listFile = new BufferedReader(new InputStreamReader(is, "UTF-8"));
         String str = "";
         // Read the list of tests to skip, compare to testName
         while ( (str = listFile.readLine()) != null )

Modified: db/derby/code/trunk/java/tools/org/apache/derby/iapi/tools/i18n/LocalizedResource.java
URL: http://svn.apache.org/viewcvs/db/derby/code/trunk/java/tools/org/apache/derby/iapi/tools/i18n/LocalizedResource.java?rev=394770&r1=394769&r2=394770&view=diff
==============================================================================
--- db/derby/code/trunk/java/tools/org/apache/derby/iapi/tools/i18n/LocalizedResource.java
(original)
+++ db/derby/code/trunk/java/tools/org/apache/derby/iapi/tools/i18n/LocalizedResource.java
Mon Apr 17 13:30:21 2006
@@ -212,6 +212,7 @@
 		timeSize=len;
 
 	}
+
 	public LocalizedInput getNewInput(InputStream i) {
 		try {
 			if (encode != null)
@@ -222,6 +223,16 @@
 		}
 		return new LocalizedInput(i);
 	}
+
+	public LocalizedInput getNewEncodedInput(InputStream i, String encoding) {
+		try {
+	          return new LocalizedInput(i,encoding);
+		}
+		catch (UnsupportedEncodingException e){
+			
+		}
+		return new LocalizedInput(i);
+        }
 
 	public LocalizedOutput getNewOutput(OutputStream o){
 		try {

Modified: db/derby/code/trunk/java/tools/org/apache/derby/impl/tools/ij/utilMain.java
URL: http://svn.apache.org/viewcvs/db/derby/code/trunk/java/tools/org/apache/derby/impl/tools/ij/utilMain.java?rev=394770&r1=394769&r2=394770&view=diff
==============================================================================
--- db/derby/code/trunk/java/tools/org/apache/derby/impl/tools/ij/utilMain.java (original)
+++ db/derby/code/trunk/java/tools/org/apache/derby/impl/tools/ij/utilMain.java Mon Apr 17
13:30:21 2006
@@ -541,7 +541,7 @@
 		if (is==null) throw ijException.resourceNotFound();
 		oldGrabbers.push(commandGrabber[currCE]);
 	    commandGrabber[currCE] = 
-                new StatementFinder(langUtil.getNewInput(new BufferedInputStream(is, BUFFEREDFILESIZE)));
+                new StatementFinder(langUtil.getNewEncodedInput(new BufferedInputStream(is,
BUFFEREDFILESIZE), "UTF8"));
 		fileInput = true;
 	}
 



Mime
View raw message