tomcat-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cos...@apache.org
Subject cvs commit: jakarta-tomcat/src/share/org/apache/tomcat/util/buf UTF8Decoder.java B2CConverter.java
Date Sat, 26 May 2001 17:12:05 GMT
costin      01/05/26 10:12:05

  Added:       src/share/org/apache/tomcat/util/buf UTF8Decoder.java
                        B2CConverter.java
  Log:
  Added a generic Byte->Char (charset) decoder, using the same model as
  the C2BConverter used by OutputBuffer.
  
  The input system is still very inefficient, this is just the first step
  ( not used in active code - I did a lot of tests but it's better to be
  safe for now, and make sure we don't have regressions with the other changes
  before activating this one ).
  
  UTF8Decoder used to be part of ByteChunk, it was fixed and tested with
  multi-byte chars.
  
  The decoding was refactored for 3 reasons:
  1. Performance. This way we can optimize individual converters and plug
  specialized versions ( like UTF8 )
  2. We should be able to use ( when detected ) nio converters
  3. Clarity. It's ( IMHO ) easier to follow the Chunk code without all the
  extra ( and complex ) decoding code in.
  
  Both clases are not in use in the current code, but should be enabled
  for 3.3 - the code is stable and efficient.
  
  Revision  Changes    Path
  1.1                  jakarta-tomcat/src/share/org/apache/tomcat/util/buf/UTF8Decoder.java
  
  Index: UTF8Decoder.java
  ===================================================================
  /*
   * ====================================================================
   *
   * The Apache Software License, Version 1.1
   *
   * Copyright (c) 1999 The Apache Software Foundation.  All rights 
   * reserved.
   *
   * Redistribution and use in source and binary forms, with or without
   * modification, are permitted provided that the following conditions
   * are met:
   *
   * 1. Redistributions of source code must retain the above copyright
   *    notice, this list of conditions and the following disclaimer. 
   *
   * 2. Redistributions in binary form must reproduce the above copyright
   *    notice, this list of conditions and the following disclaimer in
   *    the documentation and/or other materials provided with the
   *    distribution.
   *
   * 3. The end-user documentation included with the redistribution, if
   *    any, must include the following acknowlegement:  
   *       "This product includes software developed by the 
   *        Apache Software Foundation (http://www.apache.org/)."
   *    Alternately, this acknowlegement may appear in the software itself,
   *    if and wherever such third-party acknowlegements normally appear.
   *
   * 4. The names "The Jakarta Project", "Tomcat", and "Apache Software
   *    Foundation" must not be used to endorse or promote products derived
   *    from this software without prior written permission. For written 
   *    permission, please contact apache@apache.org.
   *
   * 5. Products derived from this software may not be called "Apache"
   *    nor may "Apache" appear in their names without prior written
   *    permission of the Apache Group.
   *
   * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
   * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
   * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
   * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
   * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
   * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
   * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   * SUCH DAMAGE.
   * ====================================================================
   *
   * This software consists of voluntary contributions made by many
   * individuals on behalf of the Apache Software Foundation.  For more
   * information on the Apache Software Foundation, please see
   * <http://www.apache.org/>.
   *
   * [Additional notices, if required by prior licensing conditions]
   *
   */ 
  
  package org.apache.tomcat.util.buf;
  
  import java.text.*;
  import java.util.*;
  import java.io.Serializable;
  import java.io.IOException;
  import org.apache.tomcat.util.buf.*;
  
  /**
   * Moved from ByteChunk - code to convert from UTF8 bytes to chars.
   * Not used in the current tomcat3.3 : the performance gain is not very
   * big if the String is created, only if we avoid that and work only
   * on char[]. Until than, it's better to be safe. ( I tested this code
   * with 2 and 3 bytes chars, and it works fine in xerces )
   * 
   * Cut from xerces' UTF8Reader.copyMultiByteCharData() 
   *
   * @author Costin Manolache
   * @author ( Xml-Xerces )
   */
  public final class UTF8Decoder extends B2CConverter {
      // may have state !!
      
      public UTF8Decoder() {
  
      }
      
      public void recycle() {
      }
  
      public void convert(ByteChunk mb, CharChunk cb )
  	throws IOException
      {
  	int bytesOff=mb.getOffset();
  	int bytesLen=mb.getLength();
  	byte bytes[]=mb.getBytes();
  	
  	int j=bytesOff;
  	int end=j+bytesLen;
  
  	while( j< end ) {
  	    int b0=0xff & bytes[j];
  
  	    if( (b0 & 0x80) == 0 ) {
  		cb.append((char)b0);
  		j++;
  		continue;
  	    }
  	    
  	    // 2 byte ?
  	    if( j++ >= end ) {
  		// ok, just ignore - we could throw exception
  		throw new IOException( "Conversion error - EOF " );
  	    }
  	    int b1=0xff & bytes[j];
  	    
  	    // ok, let's the fun begin - we're handling UTF8
  	    if ((0xe0 & b0) == 0xc0) { // 110yyyyy 10xxxxxx (0x80 to 0x7ff)
  		int ch = ((0x1f & b0)<<6) + (0x3f & b1);
  		if(debug>0)
  		    log("Convert " + b0 + " " + b1 + " " + ch + ((char)ch));
  		
  		cb.append((char)ch);
  		j++;
  		continue;
  	    }
  	    
  	    if( j++ >= end ) 
  		return ;
  	    int b2=0xff & bytes[j];
  	    
  	    if( (b0 & 0xf0 ) == 0xe0 ) {
  		if ((b0 == 0xED && b1 >= 0xA0) ||
  		    (b0 == 0xEF && b1 == 0xBF && b2 >= 0xBE)) {
  		    if(debug>0)
  			log("Error " + b0 + " " + b1+ " " + b2 );
  
  		    throw new IOException( "Conversion error 2"); 
  		}
  
  		int ch = ((0x0f & b0)<<12) + ((0x3f & b1)<<6) + (0x3f & b2);
  		cb.append((char)ch);
  		if(debug>0)
  		    log("Convert " + b0 + " " + b1+ " " + b2 + " " + ch +
  			((char)ch));
  		j++;
  		continue;
  	    }
  
  	    if( j++ >= end ) 
  		return ;
  	    int b3=0xff & bytes[j];
  
  	    if (( 0xf8 & b0 ) == 0xf0 ) {
  		if (b0 > 0xF4 || (b0 == 0xF4 && b1 >= 0x90)) {
  		    if(debug>0)
  			log("Convert " + b0 + " " + b1+ " " + b2 + " " + b3);
  		    throw new IOException( "Conversion error ");
  		}
  		int ch = ((0x0f & b0)<<18) + ((0x3f & b1)<<12) +
  		    ((0x3f & b2)<<6) + (0x3f & b3);
  
  		if(debug>0)
  		    log("Convert " + b0 + " " + b1+ " " + b2 + " " + b3 + " " +
  			ch + ((char)ch));
  
  		if (ch < 0x10000) {
  		    cb.append( (char)ch );
  		} else {
  		    cb.append((char)(((ch-0x00010000)>>10)+
  						   0xd800));
  		    cb.append((char)(((ch-0x00010000)&0x3ff)+
  						   0xdc00));
  		}
  		j++;
  		continue;
  	    } else {
  		// XXX Throw conversion exception !!!
  		if(debug>0)
  		    log("Convert " + b0 + " " + b1+ " " + b2 + " " + b3);
  		throw new IOException( "Conversion error 4" );
  	    }
  	}
      }
  
      private static int debug=1;
      void log(String s ) {
  	System.out.println("UTF8Decoder: " + s );
      }
      
  }
  
  
  
  1.1                  jakarta-tomcat/src/share/org/apache/tomcat/util/buf/B2CConverter.java
  
  Index: B2CConverter.java
  ===================================================================
  /*
   * ====================================================================
   *
   * The Apache Software License, Version 1.1
   *
   * Copyright (c) 1999 The Apache Software Foundation.  All rights 
   * reserved.
   *
   * Redistribution and use in source and binary forms, with or without
   * modification, are permitted provided that the following conditions
   * are met:
   *
   * 1. Redistributions of source code must retain the above copyright
   *    notice, this list of conditions and the following disclaimer. 
   *
   * 2. Redistributions in binary form must reproduce the above copyright
   *    notice, this list of conditions and the following disclaimer in
   *    the documentation and/or other materials provided with the
   *    distribution.
   *
   * 3. The end-user documentation included with the redistribution, if
   *    any, must include the following acknowlegement:  
   *       "This product includes software developed by the 
   *        Apache Software Foundation (http://www.apache.org/)."
   *    Alternately, this acknowlegement may appear in the software itself,
   *    if and wherever such third-party acknowlegements normally appear.
   *
   * 4. The names "The Jakarta Project", "Tomcat", and "Apache Software
   *    Foundation" must not be used to endorse or promote products derived
   *    from this software without prior written permission. For written 
   *    permission, please contact apache@apache.org.
   *
   * 5. Products derived from this software may not be called "Apache"
   *    nor may "Apache" appear in their names without prior written
   *    permission of the Apache Group.
   *
   * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
   * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
   * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
   * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
   * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
   * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
   * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   * SUCH DAMAGE.
   * ====================================================================
   *
   * This software consists of voluntary contributions made by many
   * individuals on behalf of the Apache Software Foundation.  For more
   * information on the Apache Software Foundation, please see
   * <http://www.apache.org/>.
   *
   * [Additional notices, if required by prior licensing conditions]
   *
   */ 
  
  
  package org.apache.tomcat.util.buf;
  
  import org.apache.tomcat.util.buf.*;
  
  import java.io.*;
  
  /** Efficient conversion of bytes  to character .
   *  
   *  This uses the standard JDK mechansim - a reader - but provides mechanisms
   *  to recycle all the objects that are used. It is compatible with JDK1.1
   *  and up,
   *  ( nio is better, but it's not available even in 1.2 or 1.3 )
   *
   *  Not used in the current code, the performance gain is not very big
   *  in the current case ( since String is created anyway ), but it will
   *  be used in a later version or after the remaining optimizations.
   */
  public class B2CConverter {
      private IntermediateInputStream iis;
      private ReadConvertor conv;
      private String encoding;
  
      protected B2CConverter() {
      }
      
      /** Create a converter, with bytes going to a byte buffer
       */
      public B2CConverter(String encoding)
  	throws IOException
      {
  	this.encoding=encoding;
  	reset();
      }
  
      
      /** Reset the internal state, empty the buffers.
       *  The encoding remain in effect, the internal buffers remain allocated.
       */
      public  void recycle() {
  	conv.recycle();
      }
  
      static final int BUFFER_SIZE=8192;
      char result[]=new char[BUFFER_SIZE];
  
      /** Convert a buffer of bytes into a chars
       */
      public  void convert( ByteChunk bb, CharChunk cb )
  	throws IOException
      {
  	// Set the ByteChunk as input to the Intermediate reader
  	iis.setByteChunk( bb );
  	convert(cb);
      }
  
      private void convert(CharChunk cb)
  	throws IOException
      {
  	try {
  	    // read from the reader
  	    while( true ) { // conv.ready() ) {
  		int cnt=conv.read( result, 0, BUFFER_SIZE );
  		if( cnt <= 0 ) {
  		    // End of stream ! - we may be in a bad state
  		    if( debug>0)
  			log( "EOF" );
  		    //		    reset();
  		    return;
  		}
  		if( debug > 1 )
  		    log("Converted: " + new String( result, 0, cnt ));
  
  		// XXX go directly
  		cb.append( result, 0, cnt );
  	    }
  	} catch( IOException ex) {
  	    if( debug>0)
  		log( "Reseting the converter " + ex.toString() );
  	    reset();
  	    throw ex;
  	}
      }
  
      public void reset()
  	throws IOException
      {
  	// destroy the reader/iis
  	iis=new IntermediateInputStream();
  	conv=new ReadConvertor( iis, encoding );
      }
  
      private final int debug=0;
      void log( String s ) {
  	System.out.println("B2CConverter: " + s );
      }
  
      // -------------------- Not used - the speed improvemnt is quite small
  
      /*
      private Hashtable decoders;
      public static final boolean useNewString=false;
      public static final boolean useSpecialDecoders=true;
      private UTF8Decoder utfD;
      // private char[] conversionBuff;
      CharChunk conversionBuf;
  
  
      private  static String decodeString(ByteChunk mb, String enc)
  	throws IOException
      {
  	byte buff=mb.getBuffer();
  	int start=mb.getStart();
  	int end=mb.getEnd();
  	if( useNewString ) {
  	    if( enc==null) enc="UTF8";
  	    return new String( buff, start, end-start, enc );
  	}
  	B2CConverter b2c=null;
  	if( useSpecialDecoders &&
  	    (enc==null || "UTF8".equalsIgnoreCase(enc))) {
  	    if( utfD==null ) utfD=new UTF8Decoder();
  	    b2c=utfD;
  	}
  	if(decoders == null ) decoders=new Hashtable();
  	if( enc==null ) enc="UTF8";
  	b2c=(B2CConverter)decoders.get( enc );
  	if( b2c==null ) {
  	    if( useSpecialDecoders ) {
  		if( "UTF8".equalsIgnoreCase( enc ) ) {
  		    b2c=new UTF8Decoder();
  		}
  	    }
  	    if( b2c==null )
  		b2c=new B2CConverter( enc );
  	    decoders.put( enc, b2c );
  	}
  	if( conversionBuf==null ) conversionBuf=new CharChunk(1024);
  
  	try {
  	    conversionBuf.recycle();
  	    b2c.convert( this, conversionBuf );
  	    //System.out.println("XXX 1 " + conversionBuf );
  	    return conversionBuf.toString();
  	} catch( IOException ex ) {
  	    ex.printStackTrace();
  	    return null;
  	}
      }
  
      */
  }
  
  // -------------------- Private implementation --------------------
  
  
  
  /**
   * 
   */
  final class  ReadConvertor extends InputStreamReader {
      // stream with flush() and close(). overriden.
      private IntermediateInputStream iis;
      
      // Has a private, internal byte[8192]
      
      /** Create a converter.
       */
      public ReadConvertor( IntermediateInputStream in, String enc )
  	throws UnsupportedEncodingException
      {
  	super( in, enc );
  	iis=in;
      }
      
      /** Overriden - will do nothing but reset internal state.
       */
      public  final void close() throws IOException {
  	// NOTHING
  	// Calling super.close() would reset out and cb.
      }
      
      public  final int read(char cbuf[], int off, int len)
  	throws IOException
      {
  	// will do the conversion and call write on the output stream
  	return super.read( cbuf, off, len );
      }
      
      /** Reset the buffer
       */
      public  final void recycle() {
      }
  }
  
  
  /** Special output stream where close() is overriden, so super.close()
      is never called.
      
      This allows recycling. It can also be disabled, so callbacks will
      not be called if recycling the converter and if data was not flushed.
  */
  final class IntermediateInputStream extends InputStream {
      byte buf[];
      int pos;
      int len;
      int end;
      
      public IntermediateInputStream() {
      }
      
      public  final void close() throws IOException {
  	// shouldn't be called - we filter it out in writer
  	throw new IOException("close() called - shouldn't happen ");
      }
      
      public  final  int read(byte cbuf[], int off, int len) throws IOException {
  	if( pos >= end ) return -1;
  	if (pos + len > end) {
  	    len = end - pos;
  	}
  	if (len <= 0) {
  	    return 0;
  	}
  	System.arraycopy(buf, pos, cbuf, off, len);
  	pos += len;
  	return len;
      }
      
      public  final int read() throws IOException {
  	return (pos < end ) ? (buf[pos++] & 0xff) : -1;
      }
  
      // -------------------- Internal methods --------------------
  
      void setBuffer( byte b[], int p, int l ) {
  	buf=b;
  	pos=p;
  	len=l;
  	end=pos+len;
      }
  
      void setByteChunk( ByteChunk mb ) {
  	buf=mb.getBytes();
  	pos=mb.getStart();
  	len=mb.getLength();
  	end=pos+len;
      }
  
  }
  
  
  

Mime
View raw message