001    /* Copyright 2000, 2001, Compaq Computer Corporation */
002    
003    package javafe.parser;
004    
005    import javafe.ast.Identifier;
006    import javafe.ast.LexicalPragma;
007    import javafe.ast.ModifierPragma;
008    import javafe.ast.TypeModifierPragma;
009    import javafe.ast.StmtPragma;
010    import javafe.ast.TypeDeclElemPragma;
011    import javafe.ast.PrettyPrint;
012    
013    import javafe.util.Assert;
014    import javafe.util.Location;
015    
016    /**
017     * The <code>Token</code> class defines a set of fields that describe
018     * lexical tokens.
019     *
020     * <p> Allocating an individual <code>Token</code> object for every
021     * token consumes a lot of memory.  Instead, our front end tries to
022     * reuse <code>Token</code> instances as much as possible.  Thus, for
023     * example, <code>Lex</code> is a subclass of <code>Token</code>;
024     * <code>Lex</code> returns information about the current token not by
025     * allocating a new <code>Token</code> object but rather by filling in
026     * the <code>Token</code> fields of itself.  To facilitate the reuse
027     * of <code>Token</code> instances, <code>Token</code> has a
028     * <code>copyInto</code> method.
029     */
030    
031    public class Token
032    {
033        /***************************************************
034         *                                                 *
035         * Instance fields:                                *
036         *                                                 *
037         **************************************************/
038    
039        /**
040         * Integer code giving the kind of token.<p>
041         *
042         * To clear a token, set this field to Token.CLEAR and set
043         * identifierVal and auxVal to null, as is done in the
044         * clear() method.
045         *
046         * @see javafe.parser.TagConstants
047         */
048        public int ttype = CLEAR;
049    
050        //* The token code to use to clear a token; EOF for now.
051        public static final int CLEAR = TagConstants.EOF;
052    
053        public void clear() {
054            ttype = CLEAR;
055            identifierVal = null;
056            auxVal = null;
057        }
058    
059        /** The location of the first character of the token. */
060        //@ invariant startingLoc != Location.NULL;
061        public int startingLoc;
062    
063        /**
064         * The location of the last character of the token.  (This value
065         * isn't "off-by-one" right now.)
066         */
067        //@ invariant endingLoc != Location.NULL;
068        public int endingLoc;
069    
070    
071        /**
072         * Identifier represented by the token.  Must be non-null if
073         * <code>ttype</code> is <TT>TagConstants.IDENT</TT>.
074         */
075        /*@ invariant (ttype==TagConstants.IDENT) ==> (identifierVal != null); */
076        public Identifier identifierVal;
077    
078        /**
079         * Auxillary information about the token.  In the case of literal
080         * tokens, this field holds the value of token.  In particular,
081         * if <code>ttype</code> is one of the codes on the left of the
082         * following table, then <code>auxVal</code> must be an instance
083         * of the class on the right:
084         *
085         * <center><code><table>
086         *    <tr><td> TagConstants.INTLIT </td>    <td> Integer </td></tr>
087         *    <tr><td> TagConstants.LONGLIT </td>   <td> Long </td></tr>
088         *    <tr><td> TagConstants.FLOATLIT </td>  <td> Float </td></tr>
089         *    <tr><td> TagConstants.DOUBLELIT </td> <td> Double </td></tr>
090         *    <tr><td> TagConstants.STRINGLIT </td> <td> String </td></tr>
091         *    <tr><td> TagConstants.CHARLIT </td>   <td> Integer </td></tr>
092         *    <tr><td> TagConstants.LEXICALPRAGMA </td>
093         *                                <td> LexicalPragma </td></tr>
094         *    <tr><td> TagConstants.MODIFIERPRAGMA </td>
095         *                                <td> ModifierPragma </td></tr>
096         *    <tr><td> TagConstants.STMTPRAGMA </td> <td> StmtPragma </td></tr>
097         *    <tr><td> TagConstants.TYPEDECLELEMPRAGMA </td>
098         *                                <td> TypeDeclElemPragma</td></tr>
099         * </table> </code> </center><p>
100         *
101         * For the various pragmas, <code>auxVal</code> may be
102         * <code>null</code>, but for the literals it may <em>not</em>
103         * be.
104         */
105        /*@ invariant (
106         !(ttype==TagConstants.BOOLEANLIT) &&
107         (ttype==TagConstants.INTLIT ==> auxVal instanceof Integer) &&
108         (ttype==TagConstants.LONGLIT ==> auxVal instanceof Long) &&
109         (ttype==TagConstants.FLOATLIT ==> auxVal instanceof Float) &&
110         (ttype==TagConstants.DOUBLELIT ==> auxVal instanceof Double) &&
111         (ttype==TagConstants.STRINGLIT ==> auxVal instanceof String) &&
112         (ttype==TagConstants.CHARLIT ==> auxVal instanceof Integer) &&
113         (ttype==TagConstants.LEXICALPRAGMA ==> auxVal instanceof
114         LexicalPragma) &&
115         (ttype==TagConstants.MODIFIERPRAGMA ==> auxVal instanceof
116         ModifierPragma) &&
117         (ttype==TagConstants.STMTPRAGMA ==> auxVal instanceof StmtPragma) &&
118         (ttype==TagConstants.TYPEDECLELEMPRAGMA ==> auxVal instanceof
119         TypeDeclElemPragma) &&
120         (ttype==TagConstants.TYPEMODIFIERPRAGMA ==> auxVal instanceof
121         TypeModifierPragma)
122         ); */
123        public Object auxVal;
124    
125    
126        /***************************************************
127         *                                                 *
128         * Creation:                                       *
129         *                                                 *
130         **************************************************/
131    
132        /*
133         * This is not really safe since violates invariants... !!!!
134         *
135         * NOTE: This is not a helper; we use invalid tokens in TokenQueue and
136         *       Lex.savedState.
137         */
138        public Token() {}   //@ nowarn Invariant;  // produces invalid Token
139    
140    
141        /**
142         * Copy all the fields of <code>this</code> into
143         * <code>dst</code>.  For convenience, returns <code>dst</code>.
144         */
145        public final Token copyInto(/*@ non_null @*/ Token dst) {
146            dst.ttype = ttype;
147            dst.startingLoc = startingLoc;
148            dst.endingLoc = endingLoc;
149            dst.identifierVal = identifierVal;
150            dst.auxVal = auxVal;
151            return dst;
152        }
153    
154    
155        /***************************************************
156         *                                                 *
157         * Operations:                                     *
158         *                                                 *
159         **************************************************/
160    
161        /**
162         * Return a representation of <code>this</code> suitable for debug
163         * output.
164         */
165        public String ztoString() {
166            String result = Location.toFileName(startingLoc);
167            if (! Location.isWholeFileLoc(startingLoc))
168                result += ":" + Location.toOffset(startingLoc);
169            result += ": ";
170            if (ttype == TagConstants.IDENT)
171                result += "IDENT (" + identifierVal.toString() + ")";
172            else if (ttype == TagConstants.CHARLIT || ttype == TagConstants.INTLIT
173                     || ttype == TagConstants.LONGLIT
174                     || ttype == TagConstants.FLOATLIT
175                     || ttype == TagConstants.DOUBLELIT
176                     || ttype == TagConstants.STRINGLIT)
177                result += (PrettyPrint.inst.toString(ttype) + " ("
178                           + PrettyPrint.toCanonicalString(ttype, auxVal) + ")");
179            else if (ttype == TagConstants.LEXICALPRAGMA
180                     || ttype == TagConstants.MODIFIERPRAGMA
181                     || ttype == TagConstants.TYPEDECLELEMPRAGMA
182                     || ttype == TagConstants.STMTPRAGMA)
183                result += PrettyPrint.inst.toString(ttype) + " (" + auxVal.toString() + ")";
184            else result += PrettyPrint.inst.toString(ttype);
185            return result;
186        }
187    
188    
189        /** Check the invariants of <code>this</code>. */
190    
191        public void zzz() {
192            Assert.notFalse(ttype != TagConstants.IDENT
193                            || identifierVal != null);
194            Assert.notFalse(ttype != TagConstants.INTLIT
195                            || auxVal instanceof Integer);
196            Assert.notFalse(ttype != TagConstants.LONGLIT
197                            || auxVal instanceof Long);
198            Assert.notFalse(ttype != TagConstants.FLOATLIT
199                            || auxVal instanceof Float);
200            Assert.notFalse(ttype != TagConstants.DOUBLELIT
201                            || auxVal instanceof Double);
202            Assert.notFalse(ttype != TagConstants.STRINGLIT
203                            || auxVal instanceof String);
204            Assert.notFalse(ttype != TagConstants.CHARLIT
205                            || auxVal instanceof Integer);
206            Assert.notFalse(ttype != TagConstants.LEXICALPRAGMA
207                            || auxVal == null || auxVal instanceof LexicalPragma);
208            Assert.notFalse(ttype != TagConstants.MODIFIERPRAGMA
209                            || auxVal == null || auxVal instanceof ModifierPragma);
210            Assert.notFalse(ttype != TagConstants.STMTPRAGMA
211                            || auxVal == null || auxVal instanceof StmtPragma);
212            Assert.notFalse(ttype != TagConstants.TYPEDECLELEMPRAGMA
213                            || auxVal == null || auxVal instanceof TypeDeclElemPragma);
214        }
215    }