lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject cvs commit: jakarta-lucene/src/java/org/apache/lucene/analysis/standard StandardTokenizer.java StandardTokenizer.jj StandardTokenizerConstants.java StandardTokenizerTokenManager.java
Date Mon, 22 Dec 2003 22:12:24 GMT
cutting     2003/12/22 14:12:24

  Modified:    .        CHANGES.txt
               src/java/org/apache/lucene/analysis/standard
                        StandardTokenizer.java StandardTokenizer.jj
                        StandardTokenizerConstants.java
                        StandardTokenizerTokenManager.java
  Log:
  Fix StandardTokenizer's handling of CJK characters.
  
  Revision  Changes    Path
  1.64      +6 -1      jakarta-lucene/CHANGES.txt
  
  Index: CHANGES.txt
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/CHANGES.txt,v
  retrieving revision 1.63
  retrieving revision 1.64
  diff -u -r1.63 -r1.64
  --- CHANGES.txt	22 Dec 2003 21:42:48 -0000	1.63
  +++ CHANGES.txt	22 Dec 2003 22:12:24 -0000	1.64
  @@ -21,6 +21,11 @@
       than the final token position.  Position is used in phrase
       searching (see PhraseQuery and Token.setPositionIncrement()).
   
  + 5. Fix StandardTokenizer's handling of CJK characters (Chinese,
  +    Japanese and Korean ideograms).  Previously contiguous sequences
  +    were combined in a single token, which is not very useful.  Now
  +    each ideogram generates a separate token, which is more useful.
  +
   
   1.3 RC3
   
  
  
  
  1.3       +6 -6      jakarta-lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
  
  Index: StandardTokenizer.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java,v
  retrieving revision 1.2
  retrieving revision 1.3
  diff -u -r1.2 -r1.3
  --- StandardTokenizer.java	1 Oct 2003 16:39:26 -0000	1.2
  +++ StandardTokenizer.java	22 Dec 2003 22:12:24 -0000	1.3
  @@ -47,8 +47,8 @@
       case NUM:
         token = jj_consume_token(NUM);
         break;
  -    case SIGRAM:
  -      token = jj_consume_token(SIGRAM);
  +    case CJK:
  +      token = jj_consume_token(CJK);
         break;
       case 0:
         token = jj_consume_token(0);
  @@ -79,7 +79,7 @@
         jj_la1_0();
      }
      private static void jj_la1_0() {
  -      jj_la1_0 = new int[] {0x4ff,};
  +      jj_la1_0 = new int[] {0x10ff,};
      }
   
     public StandardTokenizer(CharStream stream) {
  @@ -158,8 +158,8 @@
   
     public ParseException generateParseException() {
       jj_expentries.removeAllElements();
  -    boolean[] la1tokens = new boolean[16];
  -    for (int i = 0; i < 16; i++) {
  +    boolean[] la1tokens = new boolean[15];
  +    for (int i = 0; i < 15; i++) {
         la1tokens[i] = false;
       }
       if (jj_kind >= 0) {
  @@ -175,7 +175,7 @@
           }
         }
       }
  -    for (int i = 0; i < 16; i++) {
  +    for (int i = 0; i < 15; i++) {
         if (la1tokens[i]) {
           jj_expentry = new int[1];
           jj_expentry[0] = i;
  
  
  
  1.6       +2 -3      jakarta-lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.jj
  
  Index: StandardTokenizer.jj
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.jj,v
  retrieving revision 1.5
  retrieving revision 1.6
  diff -u -r1.5 -r1.6
  --- StandardTokenizer.jj	9 Oct 2003 12:24:11 -0000	1.5
  +++ StandardTokenizer.jj	22 Dec 2003 22:12:24 -0000	1.6
  @@ -125,7 +125,6 @@
       (<LETTER>|<DIGIT>)*
     >
   
  -| < SIGRAM: (<CJK>)+ >
   | < #ALPHA: (<LETTER>)+>
   | < #LETTER:					  // unicode letters
         [
  @@ -137,7 +136,7 @@
          "\u0100"-"\u1fff"
         ]
     >
  -| < #CJK:             // non-alphabets
  +| < CJK:                                          // non-alphabets
         [
          "\u3040"-"\u318f",
          "\u3300"-"\u337f",
  @@ -187,7 +186,7 @@
       token = <EMAIL> |
       token = <HOST> |
       token = <NUM> |
  -    token = <SIGRAM> |
  +    token = <CJK> |
       token = <EOF>
      )
       {
  
  
  
  1.3       +5 -7      jakarta-lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerConstants.java
  
  Index: StandardTokenizerConstants.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerConstants.java,v
  retrieving revision 1.2
  retrieving revision 1.3
  diff -u -r1.2 -r1.3
  --- StandardTokenizerConstants.java	1 Oct 2003 16:39:26 -0000	1.2
  +++ StandardTokenizerConstants.java	22 Dec 2003 22:12:24 -0000	1.3
  @@ -13,12 +13,11 @@
     int NUM = 7;
     int P = 8;
     int HAS_DIGIT = 9;
  -  int SIGRAM = 10;
  -  int ALPHA = 11;
  -  int LETTER = 12;
  -  int CJK = 13;
  -  int DIGIT = 14;
  -  int NOISE = 15;
  +  int ALPHA = 10;
  +  int LETTER = 11;
  +  int CJK = 12;
  +  int DIGIT = 13;
  +  int NOISE = 14;
   
     int DEFAULT = 0;
   
  @@ -33,7 +32,6 @@
       "<NUM>",
       "<P>",
       "<HAS_DIGIT>",
  -    "<SIGRAM>",
       "<ALPHA>",
       "<LETTER>",
       "<CJK>",
  
  
  
  1.4       +18 -18    jakarta-lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerTokenManager.java
  
  Index: StandardTokenizerTokenManager.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerTokenManager.java,v
  retrieving revision 1.3
  retrieving revision 1.4
  diff -u -r1.3 -r1.4
  --- StandardTokenizerTokenManager.java	9 Oct 2003 12:24:11 -0000	1.3
  +++ StandardTokenizerTokenManager.java	22 Dec 2003 22:12:24 -0000	1.4
  @@ -8,7 +8,7 @@
     public  void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
   private final int jjMoveStringLiteralDfa0_0()
   {
  -   return jjMoveNfa_0(1, 0);
  +   return jjMoveNfa_0(0, 0);
   }
   private final void jjCheckNAdd(int state)
   {
  @@ -101,7 +101,7 @@
            {
               switch(jjstateSet[--i])
               {
  -               case 1:
  +               case 0:
                     if ((0x3ff000000000000L & l) != 0L)
                     {
                        if (kind > 1)
  @@ -111,6 +111,10 @@
                     if ((0x3ff000000000000L & l) != 0L)
                        jjCheckNAddStates(18, 23);
                     break;
  +               case 1:
  +                  if ((0x3ff000000000000L & l) != 0L)
  +                     jjCheckNAddStates(18, 23);
  +                  break;
                  case 2:
                  case 39:
                     if ((0x3ff000000000000L & l) != 0L)
  @@ -380,7 +384,7 @@
            {
               switch(jjstateSet[--i])
               {
  -               case 1:
  +               case 0:
                     if ((0x7fffffe07fffffeL & l) != 0L)
                        jjCheckNAddStates(30, 35);
                     if ((0x7fffffe07fffffeL & l) != 0L)
  @@ -669,12 +673,11 @@
            {
               switch(jjstateSet[--i])
               {
  -               case 1:
  +               case 0:
                     if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                     {
  -                     if (kind > 10)
  -                        kind = 10;
  -                     jjCheckNAdd(0);
  +                     if (kind > 12)
  +                        kind = 12;
                     }
                     if (jjCanMove_1(hiByte, i1, i2, l1, l2))
                        jjCheckNAddStates(18, 23);
  @@ -687,12 +690,9 @@
                     if (jjCanMove_2(hiByte, i1, i2, l1, l2))
                        jjCheckNAddStates(30, 35);
                     break;
  -               case 0:
  -                  if (!jjCanMove_0(hiByte, i1, i2, l1, l2))
  -                     break;
  -                  if (kind > 10)
  -                     kind = 10;
  -                  jjCheckNAdd(0);
  +               case 1:
  +                  if (jjCanMove_1(hiByte, i1, i2, l1, l2))
  +                     jjCheckNAddStates(18, 23);
                     break;
                  case 2:
                     if (jjCanMove_2(hiByte, i1, i2, l1, l2))
  @@ -1021,15 +1021,15 @@
   }
   public static final String[] jjstrLiteralImages = {
   "", null, null, null, null, null, null, null, null, null, null, null, null, 
  -null, null, null, };
  +null, null, };
   public static final String[] lexStateNames = {
      "DEFAULT", 
   };
   static final long[] jjtoToken = {
  -   0x4ffL, 
  +   0x10ffL, 
   };
   static final long[] jjtoSkip = {
  -   0x8000L, 
  +   0x4000L, 
   };
   protected CharStream input_stream;
   private final int[] jjrounds = new int[73];
  @@ -1115,9 +1115,9 @@
      jjmatchedKind = 0x7fffffff;
      jjmatchedPos = 0;
      curPos = jjMoveStringLiteralDfa0_0();
  -   if (jjmatchedPos == 0 && jjmatchedKind > 15)
  +   if (jjmatchedPos == 0 && jjmatchedKind > 14)
      {
  -      jjmatchedKind = 15;
  +      jjmatchedKind = 14;
      }
      if (jjmatchedKind != 0x7fffffff)
      {
  
  
  

---------------------------------------------------------------------
To unsubscribe, e-mail: lucene-dev-unsubscribe@jakarta.apache.org
For additional commands, e-mail: lucene-dev-help@jakarta.apache.org


Mime
View raw message