1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
|
/*
* Anarres C Preprocessor
* Copyright (c) 2007-2008, Shevek
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package org.anarres.cpp;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
/**
* A Preprocessor token.
*
* @see Preprocessor
*/
public final class Token {
// public static final int EOF = -1;
private final int type;
private int line;
private int column;
private final Object value;
private final String text;
public Token(int type, int line, int column,
String text, Object value) {
this.type = type;
this.line = line;
this.column = column;
this.text = text;
this.value = value;
}
public Token(int type, int line, int column, String text) {
this(type, line, column, text, null);
}
/* pp */ Token(int type, String text, Object value) {
this(type, -1, -1, text, value);
}
/* pp */ Token(int type, String text) {
this(type, text, null);
}
/* pp */ Token(int type) {
this(type, TokenType.getTokenText(type));
}
/**
* Returns the semantic type of this token.
*
* @return the semantic type of this token.
* @see #getTokenName(int)
*/
public int getType() {
return type;
}
/* pp */ void setLocation(int line, int column) {
this.line = line;
this.column = column;
}
/**
* Returns the line at which this token started.
*
* Lines are numbered from 1.
*
* @return the line at which this token started.
* @see LexerSource#getLine()
*/
// Not @Nonnegative - might not have been assigned?
public int getLine() {
return line;
}
/**
* Returns the column at which this token started.
*
* Columns are numbered from 0.
*
* @return the column at which this token started.
* @see LexerSource#getColumn()
*/
// Not @Nonnegative - might not have been assigned?
public int getColumn() {
return column;
}
/**
* Returns the original or generated text of this token.
*
* This is distinct from the semantic value of the token.
*
* @return the original or generated text of this token.
* @see #getValue()
*/
// Not @Nonnull - might not have been assigned?
public String getText() {
return text;
}
/**
* Returns the semantic value of this token.
*
* For strings, this is the parsed String.
* For integers, this is an Integer object.
* For other token types, as appropriate.
*
* @return the semantic value of this token, or null.
* @see #getText()
*/
@CheckForNull
public Object getValue() {
return value;
}
/**
* Returns a description of this token, for debugging purposes.
*/
@Override
public String toString() {
StringBuilder buf = new StringBuilder();
buf.append('[').append(getTokenName(type));
if (line != -1) {
buf.append('@').append(line);
if (column != -1)
buf.append(',').append(column);
}
buf.append("]:");
if (text != null)
buf.append('"').append(text).append('"');
else if (type > 3 && type < 256)
buf.append((char) type);
else
buf.append('<').append(type).append('>');
if (value != null)
buf.append('=').append(value);
return buf.toString();
}
/**
* Returns the descriptive name of the given token type.
*
* This is mostly used for stringification and debugging.
*
* @param type The type constant from this class to name.
* @return the descriptive name of the given token type.
* @see Token#getType()
*/
@Nonnull
public static String getTokenName(int type) {
return TokenType.getTokenName(type);
}
public static final int AND_EQ = 257;
public static final int ARROW = 258;
public static final int CHARACTER = 259;
public static final int CCOMMENT = 260;
public static final int CPPCOMMENT = 261;
public static final int DEC = 262;
public static final int DIV_EQ = 263;
public static final int ELLIPSIS = 264;
public static final int EOF = 265;
public static final int EQ = 266;
public static final int GE = 267;
public static final int HASH = 268;
public static final int HEADER = 269;
public static final int IDENTIFIER = 270;
public static final int INC = 271;
public static final int NUMBER = 272;
public static final int LAND = 273;
public static final int LAND_EQ = 274;
public static final int LE = 275;
public static final int LITERAL = 276;
public static final int LOR = 277;
public static final int LOR_EQ = 278;
public static final int LSH = 279;
public static final int LSH_EQ = 280;
public static final int MOD_EQ = 281;
public static final int MULT_EQ = 282;
public static final int NE = 283;
public static final int NL = 284;
public static final int OR_EQ = 285;
public static final int PASTE = 286;
public static final int PLUS_EQ = 287;
public static final int RANGE = 288;
public static final int RSH = 289;
public static final int RSH_EQ = 290;
public static final int SQSTRING = 291;
public static final int STRING = 292;
public static final int SUB_EQ = 293;
public static final int WHITESPACE = 294;
public static final int XOR_EQ = 295;
public static final int M_ARG = 296;
public static final int M_PASTE = 297;
public static final int M_STRING = 298;
public static final int P_LINE = 299;
public static final int INVALID = 300;
/** The position-less space token. */
/* pp */ static final Token space = new Token(WHITESPACE, -1, -1, " ");
}
|
