    001    /* Copyright 2000, 2001, Compaq Computer Corporation */
    002
    003    package javafe.parser;
    004
    005    import javafe.ast.Identifier;
    006    import javafe.ast.LexicalPragma;
    007    import javafe.ast.ModifierPragma;
    008    import javafe.ast.TypeModifierPragma;
    009    import javafe.ast.StmtPragma;
    010    import javafe.ast.TypeDeclElemPragma;
    011    import javafe.ast.PrettyPrint;
    012
    013    import javafe.util.Assert;
    014    import javafe.util.Location;
    015
    016    /**
    017     * The <code>Token</code> class defines a set of fields that describe
    018     * lexical tokens.
    019     *
    020     * <p> Allocating an individual <code>Token</code> object for every
    021     * token consumes a lot of memory.  Instead, our front end tries to
    022     * reuse <code>Token</code> instances as much as possible.  Thus, for
    023     * example, <code>Lex</code> is a subclass of <code>Token</code>;
    024     * <code>Lex</code> returns information about the current token not by
    025     * allocating a new <code>Token</code> object but rather by filling in
    026     * the <code>Token</code> fields of itself.  To facilitate the reuse
    027     * of <code>Token</code> instances, <code>Token</code> has a
    028     * <code>copyInto</code> method.
    029     */
    030
    031    public class Token
    032    {
    033        /***************************************************
    034         *                                                 *
    035         * Instance fields:                                *
    036         *                                                 *
    037         **************************************************/
    038
    039        /**
    040         * Integer code giving the kind of token.<p>
    041         *
    042         * To clear a token, set this field to Token.CLEAR and set
    043         * identifierVal and auxVal to null, as is done in the
    044         * clear() method.
    045         *
    046         * @see javafe.parser.TagConstants
    047         */
    048        public int ttype = CLEAR;
    049
    050        //* The token code to use to clear a token; EOF for now.
    051        public static final int CLEAR = TagConstants.EOF;
    052
    053        public void clear() {
    054            ttype = CLEAR;
    055            identifierVal = null;
    056            auxVal = null;
    057        }
    058
    059        /** The location of the first character of the token. */
    060        //@ invariant startingLoc != Location.NULL;
    061        public int startingLoc;
    062
    063        /**
    064         * The location of the last character of the token.  (This value
    065         * isn't "off-by-one" right now.)
    066         */
    067        //@ invariant endingLoc != Location.NULL;
    068        public int endingLoc;
    069
    070
    071        /**
    072         * Identifier represented by the token.  Must be non-null if
    073         * <code>ttype</code> is <TT>TagConstants.IDENT</TT>.
    074         */
    075        /*@ invariant (ttype==TagConstants.IDENT) ==> (identifierVal != null); */
    076        public Identifier identifierVal;
    077
    078        /**
    079         * Auxillary information about the token.  In the case of literal
    080         * tokens, this field holds the value of token.  In particular,
    081         * if <code>ttype</code> is one of the codes on the left of the
    082         * following table, then <code>auxVal</code> must be an instance
    083         * of the class on the right:
    084         *
    085         * <center><code><table>
    086         *    <tr><td> TagConstants.INTLIT </td>    <td> Integer </td></tr>
    087         *    <tr><td> TagConstants.LONGLIT </td>   <td> Long </td></tr>
    088         *    <tr><td> TagConstants.FLOATLIT </td>  <td> Float </td></tr>
    089         *    <tr><td> TagConstants.DOUBLELIT </td> <td> Double </td></tr>
    090         *    <tr><td> TagConstants.STRINGLIT </td> <td> String </td></tr>
    091         *    <tr><td> TagConstants.CHARLIT </td>   <td> Integer </td></tr>
    092         *    <tr><td> TagConstants.LEXICALPRAGMA </td>
    093         *                                <td> LexicalPragma </td></tr>
    094         *    <tr><td> TagConstants.MODIFIERPRAGMA </td>
    095         *                                <td> ModifierPragma </td></tr>
    096         *    <tr><td> TagConstants.STMTPRAGMA </td> <td> StmtPragma </td></tr>
    097         *    <tr><td> TagConstants.TYPEDECLELEMPRAGMA </td>
    098         *                                <td> TypeDeclElemPragma</td></tr>
    099         * </table> </code> </center><p>
    100         *
    101         * For the various pragmas, <code>auxVal</code> may be
    102         * <code>null</code>, but for the literals it may <em>not</em>
    103         * be.
    104         */
    105        /*@ invariant (
    106         !(ttype==TagConstants.BOOLEANLIT) &&
    107         (ttype==TagConstants.INTLIT ==> auxVal instanceof Integer) &&
    108         (ttype==TagConstants.LONGLIT ==> auxVal instanceof Long) &&
    109         (ttype==TagConstants.FLOATLIT ==> auxVal instanceof Float) &&
    110         (ttype==TagConstants.DOUBLELIT ==> auxVal instanceof Double) &&
    111         (ttype==TagConstants.STRINGLIT ==> auxVal instanceof String) &&
    112         (ttype==TagConstants.CHARLIT ==> auxVal instanceof Integer) &&
    113         (ttype==TagConstants.LEXICALPRAGMA ==> auxVal instanceof
    114         LexicalPragma) &&
    115         (ttype==TagConstants.MODIFIERPRAGMA ==> auxVal instanceof
    116         ModifierPragma) &&
    117         (ttype==TagConstants.STMTPRAGMA ==> auxVal instanceof StmtPragma) &&
    118         (ttype==TagConstants.TYPEDECLELEMPRAGMA ==> auxVal instanceof
    119         TypeDeclElemPragma) &&
    120         (ttype==TagConstants.TYPEMODIFIERPRAGMA ==> auxVal instanceof
    121         TypeModifierPragma)
    122         ); */
    123        public Object auxVal;
    124
    125
    126        /***************************************************
    127         *                                                 *
    128         * Creation:                                       *
    129         *                                                 *
    130         **************************************************/
    131
    132        /*
    133         * This is not really safe since violates invariants... !!!!
    134         *
    135         * NOTE: This is not a helper; we use invalid tokens in TokenQueue and
    136         *       Lex.savedState.
    137         */
    138        public Token() {}   //@ nowarn Invariant;  // produces invalid Token
    139
    140
    141        /**
    142         * Copy all the fields of <code>this</code> into
    143         * <code>dst</code>.  For convenience, returns <code>dst</code>.
    144         */
    145        public final Token copyInto(/*@ non_null @*/ Token dst) {
    146            dst.ttype = ttype;
    147            dst.startingLoc = startingLoc;
    148            dst.endingLoc = endingLoc;
    149            dst.identifierVal = identifierVal;
    150            dst.auxVal = auxVal;
    151            return dst;
    152        }
    153
    154
    155        /***************************************************
    156         *                                                 *
    157         * Operations:                                     *
    158         *                                                 *
    159         **************************************************/
    160
    161        /**
    162         * Return a representation of <code>this</code> suitable for debug
    163         * output.
    164         */
    165        public String ztoString() {
    166            String result = Location.toFileName(startingLoc);
    167            if (! Location.isWholeFileLoc(startingLoc))
    168                result += ":" + Location.toOffset(startingLoc);
    169            result += ": ";
    170            if (ttype == TagConstants.IDENT)
    171                result += "IDENT (" + identifierVal.toString() + ")";
    172            else if (ttype == TagConstants.CHARLIT || ttype == TagConstants.INTLIT
    173                     || ttype == TagConstants.LONGLIT
    174                     || ttype == TagConstants.FLOATLIT
    175                     || ttype == TagConstants.DOUBLELIT
    176                     || ttype == TagConstants.STRINGLIT)
    177                result += (PrettyPrint.inst.toString(ttype) + " ("
    178                           + PrettyPrint.toCanonicalString(ttype, auxVal) + ")");
    179            else if (ttype == TagConstants.LEXICALPRAGMA
    180                     || ttype == TagConstants.MODIFIERPRAGMA
    181                     || ttype == TagConstants.TYPEDECLELEMPRAGMA
    182                     || ttype == TagConstants.STMTPRAGMA)
    183                result += PrettyPrint.inst.toString(ttype) + " (" + auxVal.toString() + ")";
    184            else result += PrettyPrint.inst.toString(ttype);
    185            return result;
    186        }
    187
    188
    189        /** Check the invariants of <code>this</code>. */
    190
    191        public void zzz() {
    192            Assert.notFalse(ttype != TagConstants.IDENT
    193                            || identifierVal != null);
    194            Assert.notFalse(ttype != TagConstants.INTLIT
    195                            || auxVal instanceof Integer);
    196            Assert.notFalse(ttype != TagConstants.LONGLIT
    197                            || auxVal instanceof Long);
    198            Assert.notFalse(ttype != TagConstants.FLOATLIT
    199                            || auxVal instanceof Float);
    200            Assert.notFalse(ttype != TagConstants.DOUBLELIT
    201                            || auxVal instanceof Double);
    202            Assert.notFalse(ttype != TagConstants.STRINGLIT
    203                            || auxVal instanceof String);
    204            Assert.notFalse(ttype != TagConstants.CHARLIT
    205                            || auxVal instanceof Integer);
    206            Assert.notFalse(ttype != TagConstants.LEXICALPRAGMA
    207                            || auxVal == null || auxVal instanceof LexicalPragma);
    208            Assert.notFalse(ttype != TagConstants.MODIFIERPRAGMA
    209                            || auxVal == null || auxVal instanceof ModifierPragma);
    210            Assert.notFalse(ttype != TagConstants.STMTPRAGMA
    211                            || auxVal == null || auxVal instanceof StmtPragma);
    212            Assert.notFalse(ttype != TagConstants.TYPEDECLELEMPRAGMA
    213                            || auxVal == null || auxVal instanceof TypeDeclElemPragma);
    214        }
    215    }