001 /* Copyright 2000, 2001, Compaq Computer Corporation */ 002 003 package javafe.parser; 004 005 import javafe.ast.Identifier; 006 import javafe.ast.LexicalPragma; 007 import javafe.ast.ModifierPragma; 008 import javafe.ast.TypeModifierPragma; 009 import javafe.ast.StmtPragma; 010 import javafe.ast.TypeDeclElemPragma; 011 import javafe.ast.PrettyPrint; 012 013 import javafe.util.Assert; 014 import javafe.util.Location; 015 016 /** 017 * The <code>Token</code> class defines a set of fields that describe 018 * lexical tokens. 019 * 020 * <p> Allocating an individual <code>Token</code> object for every 021 * token consumes a lot of memory. Instead, our front end tries to 022 * reuse <code>Token</code> instances as much as possible. Thus, for 023 * example, <code>Lex</code> is a subclass of <code>Token</code>; 024 * <code>Lex</code> returns information about the current token not by 025 * allocating a new <code>Token</code> object but rather by filling in 026 * the <code>Token</code> fields of itself. To facilitate the reuse 027 * of <code>Token</code> instances, <code>Token</code> has a 028 * <code>copyInto</code> method. 029 */ 030 031 public class Token 032 { 033 /*************************************************** 034 * * 035 * Instance fields: * 036 * * 037 **************************************************/ 038 039 /** 040 * Integer code giving the kind of token.<p> 041 * 042 * To clear a token, set this field to Token.CLEAR and set 043 * identifierVal and auxVal to null, as is done in the 044 * clear() method. 045 * 046 * @see javafe.parser.TagConstants 047 */ 048 public int ttype = CLEAR; 049 050 //* The token code to use to clear a token; EOF for now. 051 public static final int CLEAR = TagConstants.EOF; 052 053 public void clear() { 054 ttype = CLEAR; 055 identifierVal = null; 056 auxVal = null; 057 } 058 059 /** The location of the first character of the token. */ 060 //@ invariant startingLoc != Location.NULL; 061 public int startingLoc; 062 063 /** 064 * The location of the last character of the token. (This value 065 * isn't "off-by-one" right now.) 066 */ 067 //@ invariant endingLoc != Location.NULL; 068 public int endingLoc; 069 070 071 /** 072 * Identifier represented by the token. Must be non-null if 073 * <code>ttype</code> is <TT>TagConstants.IDENT</TT>. 074 */ 075 /*@ invariant (ttype==TagConstants.IDENT) ==> (identifierVal != null); */ 076 public Identifier identifierVal; 077 078 /** 079 * Auxillary information about the token. In the case of literal 080 * tokens, this field holds the value of token. In particular, 081 * if <code>ttype</code> is one of the codes on the left of the 082 * following table, then <code>auxVal</code> must be an instance 083 * of the class on the right: 084 * 085 * <center><code><table> 086 * <tr><td> TagConstants.INTLIT </td> <td> Integer </td></tr> 087 * <tr><td> TagConstants.LONGLIT </td> <td> Long </td></tr> 088 * <tr><td> TagConstants.FLOATLIT </td> <td> Float </td></tr> 089 * <tr><td> TagConstants.DOUBLELIT </td> <td> Double </td></tr> 090 * <tr><td> TagConstants.STRINGLIT </td> <td> String </td></tr> 091 * <tr><td> TagConstants.CHARLIT </td> <td> Integer </td></tr> 092 * <tr><td> TagConstants.LEXICALPRAGMA </td> 093 * <td> LexicalPragma </td></tr> 094 * <tr><td> TagConstants.MODIFIERPRAGMA </td> 095 * <td> ModifierPragma </td></tr> 096 * <tr><td> TagConstants.STMTPRAGMA </td> <td> StmtPragma </td></tr> 097 * <tr><td> TagConstants.TYPEDECLELEMPRAGMA </td> 098 * <td> TypeDeclElemPragma</td></tr> 099 * </table> </code> </center><p> 100 * 101 * For the various pragmas, <code>auxVal</code> may be 102 * <code>null</code>, but for the literals it may <em>not</em> 103 * be. 104 */ 105 /*@ invariant ( 106 !(ttype==TagConstants.BOOLEANLIT) && 107 (ttype==TagConstants.INTLIT ==> auxVal instanceof Integer) && 108 (ttype==TagConstants.LONGLIT ==> auxVal instanceof Long) && 109 (ttype==TagConstants.FLOATLIT ==> auxVal instanceof Float) && 110 (ttype==TagConstants.DOUBLELIT ==> auxVal instanceof Double) && 111 (ttype==TagConstants.STRINGLIT ==> auxVal instanceof String) && 112 (ttype==TagConstants.CHARLIT ==> auxVal instanceof Integer) && 113 (ttype==TagConstants.LEXICALPRAGMA ==> auxVal instanceof 114 LexicalPragma) && 115 (ttype==TagConstants.MODIFIERPRAGMA ==> auxVal instanceof 116 ModifierPragma) && 117 (ttype==TagConstants.STMTPRAGMA ==> auxVal instanceof StmtPragma) && 118 (ttype==TagConstants.TYPEDECLELEMPRAGMA ==> auxVal instanceof 119 TypeDeclElemPragma) && 120 (ttype==TagConstants.TYPEMODIFIERPRAGMA ==> auxVal instanceof 121 TypeModifierPragma) 122 ); */ 123 public Object auxVal; 124 125 126 /*************************************************** 127 * * 128 * Creation: * 129 * * 130 **************************************************/ 131 132 /* 133 * This is not really safe since violates invariants... !!!! 134 * 135 * NOTE: This is not a helper; we use invalid tokens in TokenQueue and 136 * Lex.savedState. 137 */ 138 public Token() {} //@ nowarn Invariant; // produces invalid Token 139 140 141 /** 142 * Copy all the fields of <code>this</code> into 143 * <code>dst</code>. For convenience, returns <code>dst</code>. 144 */ 145 public final Token copyInto(/*@ non_null @*/ Token dst) { 146 dst.ttype = ttype; 147 dst.startingLoc = startingLoc; 148 dst.endingLoc = endingLoc; 149 dst.identifierVal = identifierVal; 150 dst.auxVal = auxVal; 151 return dst; 152 } 153 154 155 /*************************************************** 156 * * 157 * Operations: * 158 * * 159 **************************************************/ 160 161 /** 162 * Return a representation of <code>this</code> suitable for debug 163 * output. 164 */ 165 public String ztoString() { 166 String result = Location.toFileName(startingLoc); 167 if (! Location.isWholeFileLoc(startingLoc)) 168 result += ":" + Location.toOffset(startingLoc); 169 result += ": "; 170 if (ttype == TagConstants.IDENT) 171 result += "IDENT (" + identifierVal.toString() + ")"; 172 else if (ttype == TagConstants.CHARLIT || ttype == TagConstants.INTLIT 173 || ttype == TagConstants.LONGLIT 174 || ttype == TagConstants.FLOATLIT 175 || ttype == TagConstants.DOUBLELIT 176 || ttype == TagConstants.STRINGLIT) 177 result += (PrettyPrint.inst.toString(ttype) + " (" 178 + PrettyPrint.toCanonicalString(ttype, auxVal) + ")"); 179 else if (ttype == TagConstants.LEXICALPRAGMA 180 || ttype == TagConstants.MODIFIERPRAGMA 181 || ttype == TagConstants.TYPEDECLELEMPRAGMA 182 || ttype == TagConstants.STMTPRAGMA) 183 result += PrettyPrint.inst.toString(ttype) + " (" + auxVal.toString() + ")"; 184 else result += PrettyPrint.inst.toString(ttype); 185 return result; 186 } 187 188 189 /** Check the invariants of <code>this</code>. */ 190 191 public void zzz() { 192 Assert.notFalse(ttype != TagConstants.IDENT 193 || identifierVal != null); 194 Assert.notFalse(ttype != TagConstants.INTLIT 195 || auxVal instanceof Integer); 196 Assert.notFalse(ttype != TagConstants.LONGLIT 197 || auxVal instanceof Long); 198 Assert.notFalse(ttype != TagConstants.FLOATLIT 199 || auxVal instanceof Float); 200 Assert.notFalse(ttype != TagConstants.DOUBLELIT 201 || auxVal instanceof Double); 202 Assert.notFalse(ttype != TagConstants.STRINGLIT 203 || auxVal instanceof String); 204 Assert.notFalse(ttype != TagConstants.CHARLIT 205 || auxVal instanceof Integer); 206 Assert.notFalse(ttype != TagConstants.LEXICALPRAGMA 207 || auxVal == null || auxVal instanceof LexicalPragma); 208 Assert.notFalse(ttype != TagConstants.MODIFIERPRAGMA 209 || auxVal == null || auxVal instanceof ModifierPragma); 210 Assert.notFalse(ttype != TagConstants.STMTPRAGMA 211 || auxVal == null || auxVal instanceof StmtPragma); 212 Assert.notFalse(ttype != TagConstants.TYPEDECLELEMPRAGMA 213 || auxVal == null || auxVal instanceof TypeDeclElemPragma); 214 } 215 }