`Tokenizer.java`

    //  FILE. . . . . /home/hak/ilt/src/ilog/rif/Tokenizer.java
    //  EDIT BY . . . Hassan Ait-Kaci
    //  ON MACHINE. . Latitude407.Ilog.Biz
    //  STARTED ON. . Wed May  3 14:37:03 2006

    package ilog.rif;

This is a skeleton tokenizer for building a java parser from the Jacc grammar specified in `RCL.grm`. It is mininal in that it relies on just enough information to make a toy parser. It is rather self-explanatory.
Author: |  | Hassan Aït-Kaci
Copyright: |  | © 2006 ILOG, Inc.
Version: |  | Last modified on Tue Sep 19 08:48:24 2006 by hak

    import java.io.Reader;
    import java.io.FileReader;
    import java.io.BufferedReader;
    import java.io.IOException;

    import ilog.language.io.StreamTokenizer;
    import ilog.language.syntax.*;

    public class Tokenizer implements FileTokenizer
    {
      int lineno;
      String file;
      Reader reader;
      StreamTokenizer st;
      boolean prompt;

      public Tokenizer ()
        {
        }

      public Tokenizer (String file) throws IOException
        {
          setFile(file);
        }

      public void setFile (String file) throws IOException
        {
          setFileName(file);
          setReader(new BufferedReader(new FileReader(file)));
        }

      public final int lineNumber()
        {
          return lineno;
        }

      public final void setReader (Reader rd)
        {
          reader = rd;
          st = new StreamTokenizer(reader);
          st.ordinaryChars("+-.(){}[]<>;,:=!&|*/%");
          st.setType('.',st.NUMERIC);
          st.quoteChar('\'');
          st.quoteChar('"');
          st.wordChars("_@?$");
        }

      public final Reader getReader ()
        {
          return reader;
        }

      public final String fileName ()
        {
          return file;
        }

      public final void setFileName (String file)
        {
          this.file = file;
        }

      final ParseNode locate (ParseNode node)
        {
          return ((ParseNode)node.setStart(st.tokenStart()).setEnd(st.tokenEnd())).setFile(file);
        }

      final ParseNode integer ()
        {
          return locate(Parser.numberToken("Value",(int)st.nval));
        }

      final ParseNode floatingPoint ()
        {
          return locate(Parser.numberToken("Value",st.nval));
        }

      final ParseNode string ()
        {
          return locate(Parser.symbolToken("Value","\""+st.sval+"\""));
        }

      final ParseNode character ()
        {
          return locate(Parser.symbolToken("Value","'"+st.sval+"'"));
        }

      final ParseNode object (String symbol)
        {
          return locate(Parser.symbolToken("Object",symbol));
        }

      final ParseNode variable (String symbol)
        {
          return locate(Parser.symbolToken("Var",symbol.substring(1)));
        }

      final ParseNode function (String symbol)
        {
          return locate(Parser.symbolToken("Fun",symbol));
        }

      final ParseNode relation (String symbol)
        {
          return locate(Parser.symbolToken("Rel",symbol));
        }

      final ParseNode literal (String s)
        {
          return locate(Parser.literalToken(s));
        }

      final ParseNode literal (int c)
        {
          return locate(Parser.literalToken(String.valueOf((char)c)));
        }

      public final ParseNode nextToken () throws IOException
        {
          ParseNode t = null;
          int tk = st.nextToken();
          int nextChar = st.peek();
          lineno = st.lineno();

          switch (tk)
            {
            case StreamTokenizer.TT_SPECIAL:
              return nextToken();
            case StreamTokenizer.TT_EOF:
              t = locate(Parser.eoi());
              break;
            case StreamTokenizer.TT_NUMBER:
              if (st.isInteger)
                t = integer();
              else
                t = floatingPoint();
              break;
            case '"':
              t = string();
              break;
            case '\'':    // NB: this does NOT check what's between single quotes
              t = character();
              break;
            case StreamTokenizer.TT_WORD:
    	  if (Lexicon.isVariable(st.sval))
                {
    	      t = variable(st.sval);
                  break;
                }
              if (Lexicon.isReserved(st.sval))
                {
    	      t = literal(st.sval);
                  break;
                }
              if (Lexicon.isFunction(st.sval))
                {
    	      t = function(st.sval);
                  break;
                }
              if (Lexicon.isRelation(st.sval))
                {
    	      t = relation(st.sval);
                  break;
                }
    	  t = object(st.sval);
              break;
            case ':':
    	  if (nextChar == '-')
    	    {
    	      t = literal(String.valueOf((char)tk)+((char)nextChar));
                  st.skipChar(true);
    	      break;
    	    }
    	  t = literal(tk);
    	  break;
            case '=': case '!': case '<': case '>':
    	case '&': case '|': case '+': case '-':
    	case '*': case '/': case '%':
    	  if (nextChar == '=' ||
    	      (tk == nextChar && (tk == '+' || tk == '-' || tk == '&' || tk == '|')))
                {
                  t = literal(String.valueOf((char)tk)+((char)nextChar));
                  st.skipChar(true);
    	      break;
                }
            default:
    	  t = literal(tk);
            }
          return t;
        }
    }

This file was generated on Thu Oct 05 10:28:30 CEST 2006 from file `Tokenizer.java`
by the `ilog.language.tools.Hilite` Java tool written by Hassan Aït-Kaci