`Tokenizer.java`

    // FILE. . . . . /home/hak/ilt/src/ilog/rif/Tokenizer.java
    // EDIT BY . . . Hassan Ait-Kaci
    // ON MACHINE. . 4j4zn71
    // STARTED ON. . Mon Apr 07 14:05:49 2008

    package ilog.rif.bld;

This is a skeleton tokenizer for building a java parser from the Jacc grammar specified in `BLD.grm`. It is minimal in that it relies only on enough information to make a basic working lexer. The code below is rather self-explanatory. The tokens are:
      EQUAL     '='
      MEMBER    '#'
      SUBCLASS  '##'
      IF        ':-'
      ARROW     '->'
      LEXSPACE  '^^'
      OPENPAR   '('
      CLOSEPAR  ')'
      OPENBRA   '['
      CLOSEBRA  ']'
      AND       'And'
      OR        'Or'
      EXISTS    'Exists'
      FORALL    'Forall'
      GROUP     'Group'
      EXTERNAL  'External'
      VARIABLE  any identifier starting with a '?'
      IDENTIFIER   any maximal length of non-special chars
      STRING    any double-quoted string
Note that a `SymSpace` is just parsed as a `IDENTIFIER:IDENTIFIER` and not as an actual IRI; i.e., the full IRI syntax is not checked, nor is the colon (`':'`) character given any special meaning.
Author: |  | Hassan Aït-Kaci
Copyright: |  | © 2006 ILOG, Inc.
Version: |  | Last modified on Fri May 16 14:51:58 2008 by hak

    import java.io.Reader;
    import java.io.FileReader;
    import java.io.BufferedReader;
    import java.io.IOException;

    import ilog.language.io.StreamTokenizer;
    import ilog.language.syntax.*;

    public class Tokenizer implements FileTokenizer
    {
      int lineno;
      String file;
      Reader reader;
      StreamTokenizer st;
      boolean prompt;

      public Tokenizer ()
        {
        }

      public Tokenizer (String file) throws IOException
        {
          setFile(file);
        }

      public void setFile (String file) throws IOException
        {
          setFileName(file);
          setReader(new BufferedReader(new FileReader(file)));
        }

      public final int lineNumber()
        {
          return lineno;
        }

      public final void setReader (Reader rd)
        {
          reader = rd;
          st = new StreamTokenizer(reader);
          st.parseNumbers(false);
          st.ordinaryChars(":->=#^()[]");
          st.quoteChar('"');
          // the following are considered letters since they may appear in
          // any of the "IDENTIFIER" parts of an IRI
          st.wordChars("?~`'!@$%&+*|_;<>;,.|/\\");
        }

      public final Reader getReader ()
        {
          return reader;
        }

      public final String fileName ()
        {
          return file;
        }

      public final void setFileName (String file)
        {
          this.file = file;
        }

      final ParseNode locate (ParseNode node)
        {
          return ((ParseNode)node.setStart(st.tokenStart())
    	                     .setEnd(st.tokenEnd()))
    	     .setFile(file);
        }

      final ParseNode string ()
        {
          return locate(Parser.symbolToken("STRING","\""+st.sval+"\""));
        }

      final ParseNode variable (String symbol)
        {
          return locate(Parser.symbolToken("VARIABLE",symbol.substring(1)));
        }

      final ParseNode identifier ()
        {
          return locate(Parser.symbolToken("IDENTIFIER",st.sval));
        }

      final ParseNode literal (String s)
        {
          return locate(Parser.literalToken(s));
        }

      public final ParseNode nextToken () throws IOException
        {
          ParseNode t = null;
          int tk = st.nextToken();
          int nextChar = st.peek();
          lineno = st.lineno();

          switch (tk)
            {
            case StreamTokenizer.TT_SPECIAL:
              return nextToken();
            case StreamTokenizer.TT_EOF:
              t = locate(Parser.eoi());
              break;
    	case '(':
    	  t = literal("OPENPAR");
    	  break;
    	case ')':
    	  t = literal("CLOSEPAR");
    	  break;
    	case '[':
    	  t = literal("OPENBRA");
    	  break;
    	case ']':
    	  t = literal("CLOSEBRA");
    	  break;
    	case '=':
    	  t = literal("EQUAL");
    	  break;
    	case '#':
    	  if (nextChar != '#')
    	    t = literal("MEMBER");
    	  else
    	    {
    	      t = literal(String.valueOf((char)tk)+((char)nextChar));
    	      st.skipChar(true);
    	      t = literal("SUBCLASS");
    	    }
    	  break;
    	case '^':
    	  if (nextChar != '^')
    	    t = literal("^");
    	  else
    	    {
    	      st.skipChar(true);
    	      t = literal("LEXSPACE");
    	    }
    	  break;
            case '-':
    	  if (nextChar != '>')
    	    t = literal("-");
    	  else
    	    {
    	      st.skipChar(true);
    	      t = literal("ARROW");
    	    }
    	  break;
            case ':':
    	  if (nextChar != '-')
    	    t = literal("COLON");
    	  else
    	    {
    	      st.skipChar(true);
    	      t = literal("IF");
    	    }
    	  break;
            case '"':
              t = string();
              break;
            case StreamTokenizer.TT_WORD:
    	  if (Lexicon.isVariable(st.sval))
                {
    	      t = variable(st.sval);
                  break;
                }
    	  if (Lexicon.isReserved(st.sval))
                {
    	      t = literal(st.sval.toUpperCase());
                  break;
                }
    	  t = identifier();
              break;
            default:
    	  t = literal(String.valueOf((char)tk));
            }

          //System.out.println("Reading token: "+t);
          return t;
        }
    }

This file was generated on Tue Oct 07 11:25:20 CEST 2008 from file `Tokenizer.java`
by the `ilog.language.tools.Hilite` Java tool written by Hassan Aït-Kaci