diff options
Diffstat (limited to 'src/java')
19 files changed, 4057 insertions, 0 deletions
diff --git a/src/java/org/anarres/cpp/Argument.java b/src/java/org/anarres/cpp/Argument.java new file mode 100644 index 0000000..da87d70 --- /dev/null +++ b/src/java/org/anarres/cpp/Argument.java @@ -0,0 +1,79 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.IOException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Iterator; + +import static org.anarres.cpp.Token.*; + +/** + * A macro argument. + * + * This encapsulates a raw and preprocessed token stream. + */ +/* pp */ class Argument extends ArrayList<Token> { + public static final int NO_ARGS = -1; + + private List<Token> expansion; + + public Argument() { + this.expansion = null; + } + + public void addToken(Token tok) { + add(tok); + } + + /* pp */ void expand(Preprocessor p) + throws IOException, + LexerException { + /* Cache expansion. */ + if (expansion == null) { + this.expansion = p.expand(this); + // System.out.println("Expanded arg " + this); + } + } + + public Iterator<Token> expansion() { + return expansion.iterator(); + } + + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append("Argument("); + // buf.append(super.toString()); + buf.append("raw=[ "); + for (int i = 0; i < size(); i++) + buf.append(get(i).getText()); + buf.append(" ];expansion=[ "); + if (expansion == null) + buf.append("null"); + else + for (int i = 0; i < expansion.size(); i++) + buf.append(expansion.get(i).getText()); + buf.append(" ])"); + return buf.toString(); + } + +} diff --git a/src/java/org/anarres/cpp/CppReader.java b/src/java/org/anarres/cpp/CppReader.java new file mode 100644 index 0000000..0aa6788 --- /dev/null +++ b/src/java/org/anarres/cpp/CppReader.java @@ -0,0 +1,147 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.File; +import java.io.IOException; +import java.io.Reader; + +import static org.anarres.cpp.Token.*; + +/** + * A Reader wrapper around the Preprocessor. + * + * This is a utility class to provide a transparent {@link Reader} + * which preprocesses the input text. + * + * @see Preprocessor + * @see Reader + */ +public class CppReader extends Reader { + + private Preprocessor cpp; + private String token; + private int idx; + + public CppReader(final Reader r) { + cpp = new Preprocessor(new LexerSource(r, true) { + @Override + public String getName() { + return "<CppReader Input@" + + System.identityHashCode(r) + ">"; + } + }); + token = ""; + idx = 0; + } + + public CppReader(Preprocessor p) { + cpp = p; + token = ""; + idx = 0; + } + + /** + * Returns the Preprocessor used by this CppReader. + */ + public Preprocessor getPreprocessor() { + return cpp; + } + + /** + * Defines the given name as a macro. + * + * This is a convnience method. + */ + public void addMacro(String name) + throws LexerException { + cpp.addMacro(name); + } + + /** + * Defines the given name as a macro. + * + * This is a convnience method. + */ + public void addMacro(String name, String value) + throws LexerException { + cpp.addMacro(name, value); + } + + private boolean refill() + throws IOException { + try { + assert cpp != null : "cpp is null : was it closed?"; + if (token == null) + return false; + while (idx >= token.length()) { + Token tok = cpp.token(); + switch (tok.getType()) { + case EOF: + token = null; + return false; + case COMMENT: + if (false) { + token = " "; + break; + } + default: + token = tok.getText(); + break; + } + idx = 0; + } + return true; + } + catch (LexerException e) { + IOException ie = new IOException(String.valueOf(e)); + ie.initCause(e); + throw ie; + } + } + + public int read() + throws IOException { + if (!refill()) + return -1; + return token.charAt(idx++); + } + + /* XXX Very slow and inefficient. */ + public int read(char cbuf[], int off, int len) + throws IOException { + if (token == null) + return -1; + for (int i = 0; i < len; i++) { + int ch = read(); + if (ch == -1) + return i; + cbuf[off + i] = (char)ch; + } + return len; + } + + public void close() + throws IOException { + cpp = null; + token = null; + } + +} diff --git a/src/java/org/anarres/cpp/FileLexerSource.java b/src/java/org/anarres/cpp/FileLexerSource.java new file mode 100644 index 0000000..9f574a0 --- /dev/null +++ b/src/java/org/anarres/cpp/FileLexerSource.java @@ -0,0 +1,74 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; + +import java.util.List; +import java.util.Iterator; + +import static org.anarres.cpp.Token.*; + +/** + * A {@link Source} which lexes a file. + * + * The input is buffered. + * + * @see Source + */ +public class FileLexerSource extends LexerSource { + private File file; + + /** + * Creates a new Source for lexing the given File. + * + * Preprocessor directives are honoured within the file. + */ + public FileLexerSource(File file) + throws IOException { + super( + new BufferedReader( + new FileReader( + file + ) + ), + true + ); + + this.file = file; + } + + @Override + /* pp */ File getFile() { + return file; + } + + @Override + /* pp */ String getName() { + return String.valueOf(file); + } + + public String toString() { + return "file " + file; + } +} diff --git a/src/java/org/anarres/cpp/FixedTokenSource.java b/src/java/org/anarres/cpp/FixedTokenSource.java new file mode 100644 index 0000000..d123f89 --- /dev/null +++ b/src/java/org/anarres/cpp/FixedTokenSource.java @@ -0,0 +1,67 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.PushbackReader; +import java.io.Reader; +import java.io.StringReader; + +import java.util.Arrays; +import java.util.List; +import java.util.Iterator; + +/* pp */ class FixedTokenSource extends Source { + private static final Token EOF = + new Token(Token.EOF, "<ts-eof>"); + + private List<Token> tokens; + private int idx; + + /* pp */ FixedTokenSource(Token... tokens) { + this.tokens = Arrays.asList(tokens); + this.idx = 0; + } + + /* pp */ FixedTokenSource(List<Token> tokens) { + this.tokens = tokens; + this.idx = 0; + } + + public Token token() + throws IOException, + LexerException { + if (idx >= tokens.size()) + return EOF; + return tokens.get(idx++); + } + + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append("constant token stream " + tokens); + Source parent = getParent(); + if (parent != null) + buf.append(" in ").append(String.valueOf(parent)); + return buf.toString(); + } +} diff --git a/src/java/org/anarres/cpp/InternalException.java b/src/java/org/anarres/cpp/InternalException.java new file mode 100644 index 0000000..d228710 --- /dev/null +++ b/src/java/org/anarres/cpp/InternalException.java @@ -0,0 +1,33 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +/** + * An internal exception. + * + * This exception is thrown when an internal state violation is + * encountered. This should never happen. If it ever happens, please + * report it as a bug. + */ +public class InternalException extends RuntimeException { + public InternalException(String msg) { + super(msg); + } +} diff --git a/src/java/org/anarres/cpp/JoinReader.java b/src/java/org/anarres/cpp/JoinReader.java new file mode 100644 index 0000000..10ec535 --- /dev/null +++ b/src/java/org/anarres/cpp/JoinReader.java @@ -0,0 +1,168 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.Reader; +import java.io.PushbackReader; +import java.io.IOException; + +/* pp */ class JoinReader extends Reader { + private Reader in; + + private boolean trigraphs; + + private int newlines; + private boolean flushnl; + private int[] unget; + private int uptr; + + public JoinReader(Reader in, boolean trigraphs) { + this.in = in; + this.trigraphs = trigraphs; + this.newlines = 0; + this.flushnl = false; + this.unget = new int[2]; + this.uptr = 0; + } + + public JoinReader(Reader in) { + this(in, false); + } + + private int __read() throws IOException { + if (uptr > 0) + return unget[--uptr]; + return in.read(); + } + + private void _unread(int c) { + if (c != -1) + unget[uptr++] = c; + } + + private int _read() throws IOException { + int c = __read(); + if (c == '?' && trigraphs) { + int d = __read(); + if (d == '?') { + int e = __read(); + switch (e) { + case '(': return '['; + case ')': return ']'; + case '<': return '{'; + case '>': return '}'; + case '=': return '#'; + case '/': return '\\'; + case '\'': return '^'; + case '!': return '|'; + case '-': return '~'; + } + _unread(e); + } + _unread(d); + } + return c; + } + + public int read() throws IOException { + if (flushnl) { + if (newlines > 0) { + newlines--; + return '\n'; + } + flushnl = false; + } + + for (;;) { + int c = _read(); + switch (c) { + case '\\': + int d = _read(); + switch (d) { + case '\n': + newlines++; + continue; + case '\r': + newlines++; + int e = _read(); + if (e != '\n') + _unread(e); + continue; + default: + _unread(d); + return c; + } + case '\r': + case '\n': + case '\u2028': + case '\u2029': + case '\u000B': + case '\u000C': + case '\u0085': + flushnl = true; + return c; + case -1: + if (newlines > 0) { + newlines--; + return '\n'; + } + default: + return c; + } + } + } + + public int read(char cbuf[], int off, int len) + throws IOException { + for (int i = 0; i < len; i++) { + int ch = read(); + if (ch == -1) + return i; + cbuf[off + i] = (char)ch; + } + return len; + } + + public void close() + throws IOException { + in.close(); + } + + public String toString() { + return "JoinReader(nl=" + newlines + ")"; + } + +/* + public static void main(String[] args) throws IOException { + FileReader f = new FileReader(new File(args[0])); + BufferedReader b = new BufferedReader(f); + JoinReader r = new JoinReader(b); + BufferedWriter w = new BufferedWriter( + new java.io.OutputStreamWriter(System.out) + ); + int c; + while ((c = r.read()) != -1) { + w.write((char)c); + } + w.close(); + } +*/ + +} diff --git a/src/java/org/anarres/cpp/LexerException.java b/src/java/org/anarres/cpp/LexerException.java new file mode 100644 index 0000000..a4b5e2e --- /dev/null +++ b/src/java/org/anarres/cpp/LexerException.java @@ -0,0 +1,35 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +/** + * A preprocessor exception. + * + * Note to users: I don't really like the name of this class. S. + */ +public class LexerException extends Exception { + public LexerException(String msg) { + super(msg); + } + + public LexerException(Throwable cause) { + super(cause); + } +} diff --git a/src/java/org/anarres/cpp/LexerSource.java b/src/java/org/anarres/cpp/LexerSource.java new file mode 100644 index 0000000..a291bff --- /dev/null +++ b/src/java/org/anarres/cpp/LexerSource.java @@ -0,0 +1,677 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.File; +import java.io.IOException; +import java.io.PushbackReader; +import java.io.Reader; +import java.util.Stack; + +import static org.anarres.cpp.Token.*; + +/** Does not handle digraphs. */ +public class LexerSource extends Source { + private static final boolean DEBUG = false; + + private PushbackReader reader; + private boolean ppvalid; + private boolean bol; + private boolean include; + + private int line; + private int column; + private int lastcolumn; + private boolean cr; + + /* ppvalid is: + * false in StringLexerSource, + * true in FileLexerSource */ + public LexerSource(Reader r, boolean ppvalid) { + this.reader = new PushbackReader(new JoinReader(r), 5); + this.ppvalid = ppvalid; + this.bol = true; + this.include = false; + + this.line = 1; + this.column = 0; + this.lastcolumn = -1; + this.cr = false; + } + + @Override + public int getLine() { + return line; + } + + public int getColumn() { + return column; + } + + /* pp */ boolean isNumbered() { + return true; + } + +/* Error handling - this lot is barely worth it. */ + + private final void _error(String msg, boolean error) + throws LexerException { + int _l = line; + int _c = column; + if (_c == 0) { + _c = lastcolumn; + _l--; + } + else { + _c--; + } + if (error) + super.error(_l, _c, msg); + else + super.warning(_l, _c, msg); + } + + private final void error(String msg) + throws LexerException { + _error(msg, true); + } + + private final void warning(String msg) + throws LexerException { + _error(msg, false); + } + +/* A flag for string handling. */ + + /* pp */ void setInclude(boolean b) { + this.include = b; + } + +/* + private boolean _isLineSeparator(int c) { + return Character.getType(c) == Character.LINE_SEPARATOR + || c == -1; + } +*/ + + /* XXX Move to JoinReader and canonicalise newlines. */ + private static final boolean isLineSeparator(int c) { + switch ((char)c) { + case '\r': + case '\n': + case '\u2028': + case '\u2029': + case '\u000B': + case '\u000C': + case '\u0085': + return true; + default: + return (c == -1); + } + } + + + private int read() throws IOException { + int c = reader.read(); + switch (c) { + case '\r': + cr = true; + line++; + lastcolumn = column; + column = 0; + break; + case '\n': + if (cr) { + cr = false; + break; + } + /* fallthrough */ + case '\u2028': + case '\u2029': + case '\u000B': + case '\u000C': + case '\u0085': + cr = false; + line++; + lastcolumn = column; + column = 0; + break; + default: + cr = false; + column++; + break; + } + +/* + if (isLineSeparator(c)) { + line++; + lastcolumn = column; + column = 0; + } + else { + column++; + } +*/ + + return c; + } + + /* You can unget AT MOST one newline. */ + private void unread(int c) + throws IOException { + if (c != -1) { + if (isLineSeparator(c)) { + line--; + column = lastcolumn; + cr = false; + } + else { + column--; + } + reader.unread(c); + } + } + + private Token ccomment() + throws IOException { + StringBuilder text = new StringBuilder("/*"); + int d; + do { + do { + d = read(); + text.append((char)d); + } while (d != '*'); + do { + d = read(); + text.append((char)d); + } while (d == '*'); + } while (d != '/'); + return new Token(COMMENT, text.toString()); + } + + private Token cppcomment() + throws IOException { + StringBuilder text = new StringBuilder("//"); + int d = read(); + while (!isLineSeparator(d)) { + text.append((char)d); + d = read(); + } + unread(d); + return new Token(COMMENT, text.toString()); + } + + private int escape(StringBuilder text) + throws IOException, + LexerException { + int d = read(); + switch (d) { + case 'a': text.append('a'); return 0x0a; + case 'b': text.append('b'); return '\b'; + case 'f': text.append('f'); return '\f'; + case 'n': text.append('n'); return '\n'; + case 'r': text.append('r'); return '\r'; + case 't': text.append('t'); return '\t'; + case 'v': text.append('v'); return 0x0b; + case '\\': text.append('\\'); return '\\'; + + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + int len = 0; + int val = 0; + do { + val = (val << 3) + Character.digit(d, 8); + text.append((char)d); + d = read(); + } while (++len < 3 && Character.digit(d, 8) != -1); + unread(d); + return val; + + case 'x': + len = 0; + val = 0; + do { + val = (val << 4) + Character.digit(d, 16); + text.append((char)d); + d = read(); + } while (++len < 2 && Character.digit(d, 16) != -1); + unread(d); + return val; + + /* Exclude two cases from the warning. */ + case '"': text.append('"'); return '"'; + case '\'': text.append('\''); return '\''; + + default: + warning("Unnecessary escape character " + (char)d); + text.append((char)d); + return d; + } + } + + private Token character() + throws IOException, + LexerException { + StringBuilder text = new StringBuilder("'"); + int d = read(); + if (d == '\\') { + text.append('\\'); + d = escape(text); + } + else if (isLineSeparator(d)) { + unread(d); + error("Unterminated character literal"); + return new Token(ERROR, text.toString(), null); + } + else if (d == '\'') { + text.append('\''); + error("Empty character literal"); + return new Token(ERROR, text.toString(), null); + } + else if (!Character.isDefined(d)) { + text.append('?'); + error("Illegal unicode character literal"); + } + else { + text.append((char)d); + } + + int e = read(); + if (e != '\'') { + unread(e); + error("Illegal character constant"); + /* XXX We could do some patching up here? */ + return new Token(ERROR, text.toString(), null); + } + text.append('\''); + /* XXX Bad cast. */ + return new Token(CHARACTER, + text.toString(), Character.valueOf((char)d)); + } + + /* XXX This strips the enclosing quotes from the + * returned value. */ + private Token string(char open, char close) + throws IOException, + LexerException { + StringBuilder text = new StringBuilder(); + text.append(open); + + StringBuilder buf = new StringBuilder(); + + for (;;) { + int c = read(); + if (c == close) { + break; + } + else if (c == '\\') { + text.append('\\'); + if (!include) { + char d = (char)escape(text); + buf.append(d); + } + } + else if (c == -1) { + unread(c); + error("End of file in string literal after " + buf); + return new Token(ERROR, text.toString(), null); + } + else if (isLineSeparator(c)) { + unread(c); + error("Unterminated string literal after " + buf); + return new Token(ERROR, text.toString(), null); + } + else { + text.append((char)c); + buf.append((char)c); + } + } + text.append(close); + return new Token(close == '>' ? HEADER : STRING, + text.toString(), buf.toString()); + } + + private void number_suffix(StringBuilder text, int d) + throws IOException { + if (d == 'U') { + text.append((char)d); + d = read(); + } + if (d == 'L') { + text.append((char)d); + } + else if (d == 'I') { + text.append((char)d); + } + else { + unread(d); + } + } + + /* We already chewed a zero, so empty is fine. */ + private Token number_octal() + throws IOException, + LexerException { + StringBuilder text = new StringBuilder("0"); + int d = read(); + long val = 0; + while (Character.digit(d, 8) != -1) { + val = (val << 3) + Character.digit(d, 8); + text.append((char)d); + d = read(); + } + number_suffix(text, d); + return new Token(INTEGER, + text.toString(), Long.valueOf(val)); + } + + /* We do not know whether know the first digit is valid. */ + private Token number_hex(char x) + throws IOException, + LexerException { + StringBuilder text = new StringBuilder("0"); + text.append(x); + int d = read(); + if (Character.digit(d, 16) == -1) { + unread(d); + error("Illegal hexadecimal constant " + (char)d); + return new Token(ERROR, text.toString(), null); + } + long val = 0; + do { + val = (val << 4) + Character.digit(d, 16); + text.append((char)d); + d = read(); + } while (Character.digit(d, 16) != -1); + number_suffix(text, d); + return new Token(INTEGER, + text.toString(), Long.valueOf(val)); + } + + /* We know we have at least one valid digit, but empty is not + * fine. */ + /* XXX This needs a complete rewrite. */ + private Token number_decimal(int c) + throws IOException, + LexerException { + StringBuilder text = new StringBuilder((char)c); + int d = c; + long val = 0; + do { + val = val * 10 + Character.digit(d, 10); + text.append((char)d); + d = read(); + } while (Character.digit(d, 10) != -1); + number_suffix(text, d); + return new Token(INTEGER, + text.toString(), Long.valueOf(val)); + } + + private Token identifier(int c) + throws IOException, + LexerException { + StringBuilder text = new StringBuilder(); + int d; + text.append((char)c); + for (;;) { + d = read(); + if (Character.isIdentifierIgnorable(d)) + ; + else if (Character.isJavaIdentifierPart(d)) + text.append((char)d); + else + break; + } + unread(d); + return new Token(IDENTIFIER, text.toString()); + } + + private Token whitespace(int c) + throws IOException, + LexerException { + StringBuilder text = new StringBuilder(); + int d; + text.append((char)c); + for (;;) { + d = read(); + if (ppvalid && isLineSeparator(d)) /* XXX Ugly. */ + break; + if (Character.isWhitespace(d)) + text.append((char)d); + else + break; + } + unread(d); + return new Token(WHITESPACE, text.toString()); + } + + /* No token processed by cond() contains a newline. */ + private Token cond(char c, int yes, int no) + throws IOException { + int d = read(); + if (c == d) + return new Token(yes); + unread(d); + return new Token(no); + } + + public Token token() + throws IOException, + LexerException { + Token tok = null; + + int _l = line; + int _c = column; + + int c = read(); + int d, e; + + switch (c) { + case '\n': + if (ppvalid) { + bol = true; + if (include) { + tok = new Token(NL, _l, _c, new String("\n")); + } + else { + int nls = 0; + do { + d = read(); + nls++; + } while (d == '\n'); + unread(d); + char[] text = new char[nls]; + for (int i = 0; i < text.length; i++) + text[i] = '\n'; + // Skip the bol = false below. + tok = new Token(NL, _l, _c, new String(text)); + } + if (DEBUG) + System.out.println("lx: Returning NL: " + tok); + return tok; + } + /* Let it be handled as whitespace. */ + break; + + case '!': + tok = cond('=', NE, '!'); + break; + + case '#': + if (bol) + tok = new Token(HASH); + else + tok = cond('#', PASTE, '#'); + break; + + case '+': + d = read(); + if (d == '+') + tok = new Token(INC); + else if (d == '=') + tok = new Token(PLUS_EQ); + else + unread(d); + break; + case '-': + d = read(); + if (d == '-') + tok = new Token(DEC); + else if (d == '=') + tok = new Token(SUB_EQ); + else if (d == '>') + tok = new Token(ARROW); + else + unread(d); + break; + + case '*': + tok = cond('=', MULT_EQ, '*'); + break; + case '/': + d = read(); + if (d == '*') + tok = ccomment(); + else if (d == '/') + tok = cppcomment(); + else if (d == '=') + tok = new Token(DIV_EQ); + else + unread(d); + break; + + case '%': + tok = cond('=', MOD_EQ, '%'); + break; + + case ':': + /* :: */ + break; + + case '<': + if (include) { + tok = string('<', '>'); + } + else { + d = read(); + if (d == '=') + tok = new Token(LE); + else if (d == '<') + tok = cond('=', LSH_EQ, LSH); + else + unread(d); + } + break; + + case '=': + tok = cond('=', EQ, '='); + break; + + case '>': + d = read(); + if (d == '=') + tok = new Token(GE); + else if (d == '>') + tok = cond('=', RSH_EQ, RSH); + else + unread(d); + break; + + case '^': + tok = cond('=', XOR_EQ, '^'); + break; + + case '|': + d = read(); + if (d == '=') + tok = new Token(OR_EQ); + else if (d == '|') + tok = cond('=', LOR_EQ, LOR); + else + unread(d); + break; + case '&': + d = read(); + if (d == '&') + tok = cond('=', LAND_EQ, LAND); + else if (d == '=') + tok = new Token(AND_EQ); + else + unread(d); + break; + + case '.': + d = read(); + if (d == '.') + tok = cond('.', ELLIPSIS, RANGE); + else + unread(d); + /* XXX decimal fraction */ + break; + + case '0': + /* octal or hex */ + d = read(); + if (d == 'x' || d == 'X') + tok = number_hex((char)d); + else { + unread(d); + tok = number_octal(); + } + break; + + case '\'': + tok = character(); + break; + + case '"': + tok = string('"', '"'); + break; + + case -1: + tok = new Token(EOF, _l, _c, "<eof>"); + break; + } + + if (tok == null) { + if (Character.isWhitespace(c)) { + tok = whitespace(c); + } + else if (Character.isDigit(c)) { + tok = number_decimal(c); + } + else if (Character.isJavaIdentifierStart(c)) { + tok = identifier(c); + } + else { + tok = new Token(c); + } + } + + bol = false; + + tok.setLocation(_l, _c); + if (DEBUG) + System.out.println("lx: Returning " + tok); + // (new Exception("here")).printStackTrace(System.out); + return tok; + } + +} diff --git a/src/java/org/anarres/cpp/Macro.java b/src/java/org/anarres/cpp/Macro.java new file mode 100644 index 0000000..0d0ae55 --- /dev/null +++ b/src/java/org/anarres/cpp/Macro.java @@ -0,0 +1,157 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * A macro object. + * + * This encapsulates a name, an argument count, and a token stream + * for replacement. The replacement token stream may contain the + * extra tokens {@link Token#M_ARG} and {@link Token#M_STRING}. + */ +public class Macro { + private String name; + /* It's an explicit decision to keep these around here. We don't + * need to; the argument token type is M_ARG and the value + * is the index. The strings themselves are only used in + * stringification of the macro, for debugging. */ + private List<String> args; + private boolean variadic; + private List<Token> tokens; + + public Macro(String name) { + this.name = name; + this.args = null; + this.variadic = false; + this.tokens = new ArrayList<Token>(); + } + + /** + * Returns the name of this macro. + */ + public String getName() { + return name; + } + + /** + * Sets the arguments to this macro. + */ + public void setArgs(List<String> args) { + this.args = args; + } + + /** + * Returns true if this is a function-like macro. + */ + public boolean isFunctionLike() { + return args != null; + } + + /** + * Returns the number of arguments to this macro. + */ + public int getArgs() { + return args.size(); + } + + /** + * Sets the variadic flag on this Macro. + */ + public void setVariadic(boolean b) { + this.variadic = b; + } + + /** + * Returns true if this is a variadic function-like macro. + */ + public boolean isVariadic() { + return variadic; + } + + /** + * Adds a token to the expansion of this macro. + */ + public void addToken(Token tok) { + this.tokens.add(tok); + } + + /** + * Adds a "paste" operator to the expansion of this macro. + * + * A paste operator causes the next token added to be pasted + * to the previous token when the macro is expanded. + * It is an error for a macro to end with a paste token. + */ + public void addPaste(Token tok) { + /* + * Given: tok0 ## tok1 + * We generate: M_PASTE, tok0, tok1 + * This extends as per a stack language: + * tok0 ## tok1 ## tok2 -> + * M_PASTE, tok0, M_PASTE, tok1, tok2 + */ + this.tokens.add(tokens.size() - 1, tok); + } + + /* pp */ List<Token> getTokens() { + return tokens; + } + + public String toString() { + StringBuilder buf = new StringBuilder(name); + if (args != null) { + buf.append('('); + Iterator<String> it = args.iterator(); + while (it.hasNext()) { + buf.append(it.next()); + if (it.hasNext()) + buf.append(", "); + else if (isVariadic()) + buf.append("..."); + } + buf.append(')'); + } + if (!tokens.isEmpty()) { + boolean paste = false; + buf.append(" => "); + for (int i = 0; i < tokens.size(); i++) { + Token tok = tokens.get(i); + if (tok.getType() == Token.M_PASTE) { + paste = true; + continue; + } + else { + buf.append(tok.getText()); + } + if (paste) { + buf.append(" #" + "# "); + paste = false; + } + // buf.append(tokens.get(i)); + } + } + return buf.toString(); + } + +} diff --git a/src/java/org/anarres/cpp/MacroTokenSource.java b/src/java/org/anarres/cpp/MacroTokenSource.java new file mode 100644 index 0000000..249afdf --- /dev/null +++ b/src/java/org/anarres/cpp/MacroTokenSource.java @@ -0,0 +1,191 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.PushbackReader; +import java.io.Reader; +import java.io.StringReader; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +import static org.anarres.cpp.Token.*; + +/* pp */ class MacroTokenSource extends Source { + private Macro macro; + private Iterator<Token> tokens; /* Pointer into the macro. */ + private List<Argument> args; /* { unexpanded, expanded } */ + private Iterator<Token> arg; /* "current expansion" */ + + /* pp */ MacroTokenSource(Macro m, List<Argument> args) { + this.macro = m; + this.tokens = m.getTokens().iterator(); + this.args = args; + this.arg = null; + } + + @Override + /* pp */ boolean isExpanding(Macro m) { + /* When we are expanding an arg, 'this' macro is not + * being expanded, and thus we may re-expand it. */ + if (/* XXX this.arg == null && */ this.macro == m) + return true; + return super.isExpanding(m); + } + + private static void escape(StringBuilder buf, CharSequence cs) { + for (int i = 0; i < cs.length(); i++) { + char c = cs.charAt(i); + switch (c) { + case '\\': + buf.append("\\\\"); + break; + case '"': + buf.append("\\\""); + break; + case '\n': + buf.append("\\n"); + break; + case '\r': + buf.append("\\r"); + break; + default: + buf.append(c); + } + } + } + + private void concat(StringBuilder buf, Argument arg) { + Iterator<Token> it = arg.iterator(); + while (it.hasNext()) { + Token tok = it.next(); + buf.append(tok.getText()); + } + } + + private Token stringify(Token pos, Argument arg) { + StringBuilder buf = new StringBuilder(); + concat(buf, arg); + StringBuilder str = new StringBuilder("\""); + escape(str, buf); + str.append('\"'); + return new Token(STRING, + pos.getLine(), pos.getColumn(), + str.toString(), buf.toString()); + } + + + /* At this point, we have consumed the first M_PASTE. + * @see Macro#addPaste(Token) */ + private void paste(Token ptok) + throws IOException, + LexerException { + StringBuilder buf = new StringBuilder(); + /* We know here that arg is null or expired, + * since we cannot paste an expanded arg. */ + + int count = 2; + for (int i = 0; i < count; i++) { + if (!tokens.hasNext()) + error(ptok.getLine(), ptok.getColumn(), + "Paste at end of expansion"); + Token tok = tokens.next(); + switch (tok.getType()) { + case M_PASTE: + /* One extra to paste, plus one because the + * paste token didn't count. */ + count += 2; + ptok = tok; + break; + case M_ARG: + int idx = ((Integer)tok.getValue()).intValue(); + concat(buf, args.get(idx)); + break; + /* XXX Test this. */ + case COMMENT: + break; + default: + buf.append(tok.getText()); + break; + } + } + + /* XXX Somewhere here, need to check that concatenation + * produces a valid token. */ + + /* Push and re-lex. */ + StringBuilder src = new StringBuilder(); + escape(src, buf); + StringLexerSource sl = new StringLexerSource(src.toString()); + + arg = new SourceIterator(sl); + } + + public Token token() + throws IOException, + LexerException { + for (;;) { + /* Deal with lexed tokens first. */ + + if (arg != null) { + if (arg.hasNext()) + return arg.next(); + arg = null; + } + + if (!tokens.hasNext()) + return new Token(EOF, -1, -1, ""); /* End of macro. */ + Token tok = tokens.next(); + int idx; + switch (tok.getType()) { + case M_STRING: + /* Use the nonexpanded arg. */ + idx = ((Integer)tok.getValue()).intValue(); + return stringify(tok, args.get(idx)); + case M_ARG: + /* Expand the arg. */ + idx = ((Integer)tok.getValue()).intValue(); + // System.out.println("Pushing arg " + args.get(idx)); + arg = args.get(idx).expansion(); + break; + case M_PASTE: + paste(tok); + break; + default: + return tok; + } + } /* for */ + } + + public String toString() { + StringBuilder buf = new StringBuilder(); + buf.append("expansion of ").append(macro.getName()); + Source parent = getParent(); + if (parent != null) + buf.append(" in ").append(String.valueOf(parent)); + return buf.toString(); + } +} diff --git a/src/java/org/anarres/cpp/Main.java b/src/java/org/anarres/cpp/Main.java new file mode 100644 index 0000000..cec7a37 --- /dev/null +++ b/src/java/org/anarres/cpp/Main.java @@ -0,0 +1,111 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.File; +import java.io.IOException; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +import static org.anarres.cpp.Token.*; + +/** + * (Currently a simple test class). + */ +public class Main { + + public static void main(String[] args) throws Exception { + List<String> path = new ArrayList<String>(); + path.add("/usr/include"); + path.add("/usr/local/include"); + path.add("/usr/lib/gcc/i686-pc-linux-gnu/4.1.2/include"); + + Source source = new FileLexerSource(new File(args[0])); + Preprocessor pp = new Preprocessor(source); + pp.setIncludePath(path); + + for (int i = 1; i < args.length; i++) { + pp.push_source(new FileLexerSource(new File(args[i])),true); + } + + Macro m = new Macro("__WORDSIZE"); + m.addToken(new Token(INTEGER, -1, -1, "32", Integer.valueOf(32))); + pp.addMacro(m); + + m = new Macro("__STDC__"); + m.addToken(new Token(INTEGER, -1, -1, "1", Integer.valueOf(1))); + pp.addMacro(m); + + try { + for (;;) { + Token tok = pp.token(); + if (tok != null && tok.getType() == Token.EOF) + break; + switch (2) { + case 0: + System.out.print(tok); + break; + case 1: + System.out.print("[" + tok.getText() + "]"); + break; + case 2: + System.out.print(tok.getText()); + break; + } + } + } + catch (Exception e) { + e.printStackTrace(); + Source s = pp.getSource(); + while (s != null) { + System.out.println(" -> " + s); + s = s.getParent(); + } + + /* + Iterator<State> it = pp.states.iterator(); + while (it.hasNext()) { + System.out.println(" -? " + it.next()); + } + */ + + } + + Map<String,Macro> macros = pp.getMacros(); + List<String> keys = new ArrayList<String>( + macros.keySet() + ); + Collections.sort(keys); + Iterator<String> mt = keys.iterator(); + while (mt.hasNext()) { + String key = mt.next(); + Macro macro = macros.get(key); + System.out.println("#" + "macro " + macro); + } + + } + +} diff --git a/src/java/org/anarres/cpp/Preprocessor.java b/src/java/org/anarres/cpp/Preprocessor.java new file mode 100644 index 0000000..c1b87d7 --- /dev/null +++ b/src/java/org/anarres/cpp/Preprocessor.java @@ -0,0 +1,1511 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.File; +import java.io.IOException; + +import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +import static org.anarres.cpp.Token.*; + +/** + * A C Preprocessor. + * The Preprocessor outputs a token stream which does not need + * re-lexing for C or C++. Alternatively, the output text may be + * reconstructed by concatenating the {@link Token#getText() text} + * values of the returned {@link Token Tokens}. (See + * {@link CppReader}, which does this.) + */ +public class Preprocessor { + private static final boolean DEBUG = false; + + public static final int FL_LINEMARKER = 1; + + private static final Macro __LINE__ = new Macro("__LINE__"); + private static final Macro __FILE__ = new Macro("__FILE__"); + + private Map<String,Macro> macros; + private Stack<State> states; + private Source source; + + private List<String> path; + private PreprocessorListener listener; + + private int flags; + + public Preprocessor(Source initial, int flags) { + this.macros = new HashMap<String,Macro>(); + macros.put(__LINE__.getName(), __LINE__); + macros.put(__FILE__.getName(), __FILE__); + this.states = new Stack<State>(); + states.push(new State()); + this.source = null; + this.path = null; + setListener(new PreprocessorListener()); + setFlags(flags); + + push_source(initial, false); + /* We need to get a \n onto the end of this somehow. */ + if ((flags & FL_LINEMARKER) != 0) + source_untoken(line_token(1, source.getName(), "\n")); + } + + public Preprocessor(Source initial) { + this(initial, 0); + } + + /** Equivalent to + * 'new Preprocessor(new {@link FileLexerSource}(file))' + */ + public Preprocessor(File file) + throws IOException { + this(new FileLexerSource(file), 0); + } + + public void setListener(PreprocessorListener listener) { + this.listener = listener; + Source s = source; + while (s != null) { + s.setListener(listener); + s = s.getParent(); + } + } + + public void setFlags(int flags) { + this.flags = flags; + } + + /** + * Handles an error. + * + * If a PreprocessorListener is installed, it receives the + * error. Otherwise, it is ignored. + */ + protected void error(Token tok, String msg) + throws LexerException { + if (listener != null) + listener.handleError(source, + tok.getLine(), tok.getColumn(), + msg); + } + + /** + * Handles a warning. + * + * If a PreprocessorListener is installed, it receives the + * warning. Otherwise, it is ignored. + */ + protected void warning(Token tok, String msg) + throws LexerException { + if (listener != null) + listener.handleError(source, + tok.getLine(), tok.getColumn(), + msg); + } + +/* + public void setSource(Source source) { + this.source = source; + } +*/ + + public void addMacro(Macro m) throws LexerException { + String name = m.getName(); + /* Already handled as a source error in macro(). */ + if ("defined".equals(name)) + throw new LexerException("Cannot redefine name 'defined'"); + macros.put(m.getName(), m); + } + + /** + * Defines the given name as a macro. + * + * This is a convnience method. + */ + public void addMacro(String name, String value) + throws LexerException { + try { + Macro m = new Macro(name); + StringLexerSource s = new StringLexerSource(value); + for (;;) { + Token tok = s.token(); + if (tok.getType() == EOF) + break; + m.addToken(tok); + } + addMacro(m); + } + catch (IOException e) { + throw new LexerException(e); + } + } + + /** + * Defines the given name as a macro. + * + * This is a convnience method. + */ + public void addMacro(String name) + throws LexerException { + addMacro(name, "1"); + } + + /** + * Sets the include path used by this Preprocessor. + */ + /* Note for future: Create an IncludeHandler? */ + public void setIncludePath(List<String> path) { + this.path = path; + } + + /** + * Returns the Map of Macros parsed during the run of this + * Preprocessor. + */ + protected Map<String,Macro> getMacros() { + return macros; + } + + +/* States */ + + private void push_state() { + State top = states.peek(); + states.push(new State(top)); + } + + private void pop_state() + throws LexerException { + State s = states.pop(); + if (states.isEmpty()) { + if (listener != null) + listener.handleError(getSource(), 0, 0, + "#" + "endif without #" + "if"); + states.push(s); + } + } + + private boolean isActive() { + State state = states.peek(); + return state.isParentActive() && state.isActive(); + } + + +/* Sources */ + + /** + * Returns the top Source on the input stack. + * + * @see Source + * @see #push_source(Source,boolean) + * @see #pop_source() + */ + protected Source getSource() { + return source; + } + + /** + * Pushes a Source onto the input stack. + * + * @see #getSource() + * @see #pop_source() + */ + protected void push_source(Source source, boolean autopop) { + source.setParent(this.source, autopop); + source.setListener(listener); + this.source = source; + if (listener != null) + listener.handleSourceChange(this.source, "push"); + } + + /** + * Pops a Source from the input stack. + * + * @see #getSource() + * @see #push_source(Source,boolean) + */ + protected void pop_source() { + this.source = this.source.getParent(); + if (listener != null) + listener.handleSourceChange(this.source, "pop"); + } + + +/* Source tokens */ + + private Token source_token; + + private Token line_token(int line, String name, String extra) { + return new Token(P_LINE, line, 0, + "#line " + line + " \"" + name + "\"" + extra, + null + ); + } + + private Token source_token() + throws IOException, + LexerException { + if (source_token != null) { + Token tok = source_token; + source_token = null; + return tok; + } + + for (;;) { + Token tok = source.token(); + if (tok.getType() == EOF && source.isAutopop()) { + // System.out.println("Autopop " + source); + Source s = source; + pop_source(); + if ((flags & FL_LINEMARKER) != 0 && s.isNumbered()) { + /* Not perfect, but ... */ + source_untoken(new Token(NL, source.getLine(), 0, "\n")); + return line_token(source.getLine(), source.getName(), ""); + } + else { + continue; + } + } + return tok; + } + } + + private void source_untoken(Token tok) { + if (this.source_token != null) + throw new IllegalStateException("Cannot return two tokens"); + this.source_token = tok; + } + + private boolean isWhite(Token tok) { + int type = tok.getType(); + return (type == WHITESPACE) || (type == COMMENT); + } + + private Token source_token_nonwhite() + throws IOException, + LexerException { + Token tok; + do { + tok = source_token(); + } while (isWhite(tok)); + return tok; + } + + /** + * Returns an NL or an EOF token. + * + * The metadata on the token will be correct, which is better + * than generating a new one. + */ + private Token source_skipline(boolean white) + throws IOException, + LexerException { + // (new Exception("skipping line")).printStackTrace(System.out); + return source.skipline(white); + } + + /* processes and expands a macro. */ + private boolean macro(Macro m, Token orig) + throws IOException, + LexerException { + Token tok; + List<Argument> args; + + // System.out.println("pp: expanding " + m); + + if (m.isFunctionLike()) { + OPEN: for (;;) { + tok = source_token(); + // System.out.println("pp: open: token is " + tok); + switch (tok.getType()) { + case WHITESPACE: /* XXX Really? */ + case COMMENT: + case NL: + break; /* continue */ + case '(': + break OPEN; + default: + source_untoken(tok); + return false; + } + } + + // tok = expanded_token_nonwhite(); + tok = source_token_nonwhite(); + + /* We either have, or we should have args. + * This deals elegantly with the case that we have + * one empty arg. */ + if (tok.getType() != ')' || m.getArgs() > 0) { + args = new ArrayList<Argument>(); + + Argument arg = new Argument(); + int depth = 0; + boolean space = false; + + ARGS: for (;;) { + // System.out.println("pp: arg: token is " + tok); + switch (tok.getType()) { + case EOF: + error(tok, "EOF in macro args"); + return false; + + case ',': + if (depth == 0) { + if (m.isVariadic() && + /* We are building the last arg. */ + args.size() == m.getArgs() - 1) { + /* Just add the comma. */ + arg.addToken(tok); + } + else { + args.add(arg); + arg = new Argument(); + } + } + else { + arg.addToken(tok); + } + space = false; + break; + case ')': + if (depth == 0) { + args.add(arg); + break ARGS; + } + else { + depth--; + arg.addToken(tok); + } + space = false; + break; + case '(': + depth++; + arg.addToken(tok); + space = false; + break; + + case WHITESPACE: + case COMMENT: + /* Avoid duplicating spaces. */ + space = true; + break; + + default: + /* Do not put space on the beginning of + * an argument token. */ + if (space && ! arg.isEmpty()) + arg.addToken(Token.space); + arg.addToken(tok); + space = false; + break; + + } + // tok = expanded_token(); + tok = source_token(); + } + /* space may still be true here, thus trailing space + * is stripped from arguments. */ + + if (args.size() != m.getArgs()) { + error(tok, + "macro " + m.getName() + + " has " + m.getArgs() + " parameters " + + "but given " + args.size() + " args"); + /* We could replay the arg tokens, but I + * note that GNU cpp does exactly what we do, + * i.e. output the macro name and chew the args. + */ + return false; + } + + for (int i = 0; i < args.size(); i++) { + args.get(i).expand(this); + } + + // System.out.println("Macro " + m + " args " + args); + } + else { + /* nargs == 0 and we (correctly) got () */ + args = null; + } + + } + else { + /* Macro without args. */ + args = null; + } + + if (m == __LINE__) { + push_source(new FixedTokenSource( + new Token[] { new Token(INTEGER, + orig.getLine(), orig.getColumn(), + String.valueOf(orig.getLine()), + Integer.valueOf(orig.getLine())) } + ), true); + } + else if (m == __FILE__) { + File file = source.getFile(); + push_source(new FixedTokenSource( + new Token[] { new Token(STRING, + orig.getLine(), orig.getColumn(), + '"'+ String.valueOf(file) +'"', + file) } + ), true); + } + else { + push_source(new MacroTokenSource(m, args), true); + } + + return true; + } + + /** + * Expands an argument. + */ + /* I'd rather this were done lazily. */ + /* pp */ List<Token> expand(List<Token> arg) + throws IOException, + LexerException { + List<Token> expansion = new ArrayList<Token>(); + boolean space = false; + + push_source(new FixedTokenSource(arg), false); + EXPANSION: for (;;) { + Token tok = expanded_token(); + switch (tok.getType()) { + case EOF: + break EXPANSION; + + case WHITESPACE: + case COMMENT: + space = true; + break; + + default: + if (space && ! expansion.isEmpty()) + expansion.add(Token.space); + expansion.add(tok); + space = false; + break; + } + } + + pop_source(); + + return expansion; + } + + /* processes a #define directive */ + private Token define() + throws IOException, + LexerException { + Token tok = source_token_nonwhite(); + if (tok.getType() != IDENTIFIER) { + error(tok, "Expected identifier"); + return source_skipline(false); + } + /* if predefined */ + + String name = tok.getText(); + if ("defined".equals(name)) { + error(tok, "Cannot redefine name 'defined'"); + return source_skipline(false); + } + + Macro m = new Macro(name); + List<String> args; + + tok = source_token(); + if (tok.getType() == '(') { + tok = source_token_nonwhite(); + if (tok.getType() != ')') { + args = new ArrayList<String>(); + ARGS: for (;;) { + switch (tok.getType()) { + case IDENTIFIER: + args.add(tok.getText()); + break; + // case ELLIPSIS: + case NL: + case EOF: + error(tok, + "Unterminated macro parameter list"); + break ARGS; + default: + source_skipline(false); + error(tok, + "error in macro parameters: " + + tok.getText()); + /* XXX return? */ + break ARGS; + } + tok = source_token_nonwhite(); + switch (tok.getType()) { + case ',': + break; + case ')': + tok = source_token_nonwhite(); + break ARGS; + case ELLIPSIS: + tok = source_token_nonwhite(); + if (tok.getType() != ')') + error(tok, + "ellipsis must be on last argument"); + m.setVariadic(true); + tok = source_token_nonwhite(); + break ARGS; + + case NL: + case EOF: + /* Do not skip line. */ + error(tok, + "Unterminated macro definition"); + break ARGS; + default: + source_skipline(false); + error(tok, + "bad token in macro parameters: " + + tok.getText()); + /* XXX return? */ + break ARGS; + } + tok = source_token_nonwhite(); + } + } + else { + tok = source_token_nonwhite(); /* Lose the ')' */ + args = Collections.emptyList(); + } + + m.setArgs(args); + } + else { + /* For searching. */ + args = Collections.emptyList(); + if (tok.getType() == COMMENT || + tok.getType() == WHITESPACE) { + tok = source_token_nonwhite(); + } + } + + /* Get an expansion for the macro, using indexOf. */ + boolean space = false; + boolean paste = false; + /* XXX UGLY: Ensure no space at start. + * Careful not to break EOF/LF from above. */ + if (isWhite(tok)) /* XXX Not sure this can ever happen now. */ + tok = source_token_nonwhite(); + int idx; + + EXPANSION: for (;;) { + switch (tok.getType()) { + case EOF: + break EXPANSION; + case NL: + break EXPANSION; + + case COMMENT: + // break; + case WHITESPACE: + if (!paste) + space = true; + break; + + case PASTE: + space = false; + paste = true; + m.addPaste(new Token(M_PASTE, + tok.getLine(), tok.getColumn(), + "#" + "#", null)); + break; + + case '#': + if (space) + m.addToken(Token.space); + space = false; + Token la = source_token_nonwhite(); + if (la.getType() == IDENTIFIER && + ((idx = args.indexOf(la.getText())) != -1)) { + m.addToken(new Token(M_STRING, + la.getLine(), la.getColumn(), + "#" + la.getText(), + Integer.valueOf(idx))); + } + else { + m.addToken(tok); + /* Allow for special processing. */ + source_untoken(la); + } + break; + + case IDENTIFIER: + if (space) + m.addToken(Token.space); + space = false; + paste = false; + idx = args.indexOf(tok.getText()); + if (idx == -1) + m.addToken(tok); + else + m.addToken(new Token(M_ARG, + tok.getLine(), tok.getColumn(), + tok.getText(), + Integer.valueOf(idx))); + break; + + default: + if (space) + m.addToken(Token.space); + space = false; + paste = false; + m.addToken(tok); + break; + } + tok = source_token(); + } + + // if (DEBUG) + // System.out.println("Defined macro " + m); + addMacro(m); + + return tok; /* NL or EOF. */ + } + + private Token undef() + throws IOException, + LexerException { + Token tok = source_token_nonwhite(); + if (tok.getType() != IDENTIFIER) { + error(tok, + "Expected identifier, not " + tok.getText()); + if (tok.getType() == NL || tok.getType() == EOF) + return tok; + } + else { + Macro m = macros.get(tok.getText()); + if (m != null) { + /* XXX error if predefined */ + macros.remove(m.getName()); + } + } + return source_skipline(true); + } + + /** + * Handles a include directive. + * + * The user may override this to provide alternate semantics + * for the include directive, for example, creating a Source + * based on a virtual file system. + */ + protected void include(File parent, int line, + String name, boolean quoted) + throws IOException, + LexerException { + if (quoted) { + File dir = parent.getParentFile(); + if (dir == null) + dir = new File("/"); + File file = new File(dir, name); + // System.err.println("Include: " + file); + if (file.exists()) { + push_source(new FileLexerSource(file), true); + return; + } + } + + if (path != null) { + for (int i = 0; i < path.size(); i++) { + File file = new File( + path.get(i) + File.separator + name + ); + if (file.exists()) { + // System.err.println("Include: " + file); + push_source(new FileLexerSource(file), true); + return; + } + } + } + + if (listener != null) + listener.handleError(getSource(), + line, 0, + "Header not found: " + name + " in " + path + ); + } + + private Token include() + throws IOException, + LexerException { + LexerSource lexer = (LexerSource)source; + try { + lexer.setInclude(true); + Token tok = token_nonwhite(); + + String name; + boolean quoted; + + if (tok.getType() == STRING) { + /* XXX Use the original text, not the value. + * Backslashes must not be treated as escapes here. */ + StringBuilder buf = new StringBuilder((String)tok.getValue()); + HEADER: for (;;) { + tok = _token(); /* Do macros but nothing else. */ + switch (tok.getType()) { + case WHITESPACE: + case COMMENT: + continue; + case STRING: + buf.append((String)tok.getValue()); + break; + case NL: + case EOF: + break HEADER; + default: + warning(tok, + "Unexpected token on #"+"include line"); + return source_skipline(false); + } + } + name = buf.toString(); + quoted = true; + } + else if (tok.getType() == HEADER) { + name = (String)tok.getValue(); + quoted = false; + tok = source_skipline(true); + } + else { + error(tok, + "Expected string or header, not " + tok.getText()); + switch (tok.getType()) { + case NL: + case EOF: + return tok; + default: + /* Only if not a NL or EOF already. */ + return source_skipline(false); + } + } + + /* Do the inclusion. */ + include(source.getFile(), tok.getLine(), name, quoted); + + /* 'tok' is the 'nl' after the include. We use it after the + * #line directive. */ + if ((flags & FL_LINEMARKER) != 0) { + source_untoken(tok); + return line_token(1, name, ""); + } + return tok; + } + finally { + lexer.setInclude(false); + } + } + + /* For #error and #warning. */ + private void error(Token pptok, boolean is_error) + throws IOException, + LexerException { + StringBuilder buf = new StringBuilder(); + buf.append('#').append(pptok.getText()).append(' '); + /* Peculiar construction to ditch first whitespace. */ + Token tok = source_token_nonwhite(); + ERROR: for (;;) { + switch (tok.getType()) { + case NL: + case EOF: + break ERROR; + default: + buf.append(tok.getText()); + break; + } + tok = source_token(); + } + if (is_error) + error(pptok, buf.toString()); + else + warning(pptok, buf.toString()); + } + + + + + /* This bypasses token() for #elif expressions. + * If we don't do this, then isActive() == false + * causes token() to simply chew the entire input line. */ + private Token expanded_token() + throws IOException, + LexerException { + for (;;) { + Token tok = source_token(); + // System.out.println("Source token is " + tok); + if (tok.getType() == IDENTIFIER) { + Macro m = macros.get(tok.getText()); + if (m == null) + return tok; + if (source.isExpanding(m)) + return tok; + if (macro(m, tok)) + continue; + } + return tok; + } + } + + private Token expanded_token_nonwhite() + throws IOException, + LexerException { + Token tok; + do { + tok = expanded_token(); + // System.out.println("expanded token is " + tok); + } while (isWhite(tok)); + return tok; + } + + + private Token expr_token = null; + + private Token expr_token() + throws IOException, + LexerException { + Token tok = expr_token; + + if (tok != null) { + // System.out.println("ungetting"); + expr_token = null; + } + else { + tok = expanded_token_nonwhite(); + // System.out.println("expt is " + tok); + + if (tok.getType() == IDENTIFIER && + tok.getText().equals("defined")) { + Token la = source_token_nonwhite(); + boolean paren = false; + if (la.getType() == '(') { + paren = true; + la = source_token_nonwhite(); + } + + // System.out.println("Core token is " + la); + + if (la.getType() != IDENTIFIER) { + error(la, + "defined() needs identifier, not " + + la.getText()); + tok = new Token(INTEGER, + la.getLine(), la.getColumn(), + "0", Integer.valueOf(0)); + } + else if (macros.containsKey(la.getText())) { + // System.out.println("Found macro"); + tok = new Token(INTEGER, + la.getLine(), la.getColumn(), + "1", Integer.valueOf(1)); + } + else { + // System.out.println("Not found macro"); + tok = new Token(INTEGER, + la.getLine(), la.getColumn(), + "0", Integer.valueOf(0)); + } + + if (paren) { + la = source_token_nonwhite(); + if (la.getType() != ')') { + expr_untoken(la); + error(la, "Missing ) in defined()"); + } + } + } + } + + // System.out.println("expr_token returns " + tok); + + return tok; + } + + private void expr_untoken(Token tok) + throws LexerException { + if (expr_token != null) + throw new InternalException( + "Cannot unget two expression tokens." + ); + expr_token = tok; + } + + private int expr_priority(Token op) { + switch (op.getType()) { + case '/': return 11; + case '%': return 11; + case '*': return 11; + case '+': return 10; + case '-': return 10; + case LSH: return 9; + case RSH: return 9; + case '<': return 8; + case '>': return 8; + case LE: return 8; + case GE: return 8; + case EQ: return 7; + case NE: return 7; + case '&': return 6; + case '^': return 5; + case '|': return 4; + case LAND: return 3; + case LOR: return 2; + case '?': return 1; + default: + // System.out.println("Unrecognised operator " + op); + return 0; + } + } + + private long expr(int priority) + throws IOException, + LexerException { + /* + System.out.flush(); + (new Exception("expr(" + priority + ") called")).printStackTrace(); + System.err.flush(); + */ + + Token tok = expr_token(); + long lhs, rhs; + + // System.out.println("Expr lhs token is " + tok); + + switch (tok.getType()) { + case '(': + lhs = expr(0); + tok = expr_token(); + if (tok.getType() != ')') { + expr_untoken(tok); + error(tok, "missing ) in expression"); + return 0; + } + break; + + case '~': lhs = ~expr(11); break; + case '!': lhs = expr(11) == 0 ? 1 : 0; break; + case '-': lhs = -expr(11); break; + case INTEGER: + lhs = ((Number)tok.getValue()).longValue(); + break; + case CHARACTER: + lhs = (long)((Character)tok.getValue()).charValue(); + break; + case IDENTIFIER: + /* XXX warn */ + lhs = 0; + break; + + default: + expr_untoken(tok); + error(tok, + "Bad token in expression: " + tok.getText()); + return 0; + } + + EXPR: for (;;) { + // System.out.println("expr: lhs is " + lhs + ", pri = " + priority); + Token op = expr_token(); + int pri = expr_priority(op); /* 0 if not a binop. */ + if (pri == 0 || priority >= pri) { + expr_untoken(op); + break EXPR; + } + rhs = expr(pri); + // System.out.println("rhs token is " + rhs); + switch (op.getType()) { + case '/': + if (rhs == 0) { + error(op, "Division by zero"); + lhs = 0; + } + else { + lhs = lhs / rhs; + } + break; + case '%': + if (rhs == 0) { + error(op, "Modulus by zero"); + lhs = 0; + } + else { + lhs = lhs % rhs; + } + break; + case '*': lhs = lhs * rhs; break; + case '+': lhs = lhs + rhs; break; + case '-': lhs = lhs - rhs; break; + case '<': lhs = lhs < rhs ? 1 : 0; break; + case '>': lhs = lhs > rhs ? 1 : 0; break; + case '&': lhs = lhs & rhs; break; + case '^': lhs = lhs ^ rhs; break; + case '|': lhs = lhs | rhs; break; + + case LSH: lhs = lhs << rhs; break; + case RSH: lhs = lhs >> rhs; break; + case LE: lhs = lhs <= rhs ? 1 : 0; break; + case GE: lhs = lhs >= rhs ? 1 : 0; break; + case EQ: lhs = lhs == rhs ? 1 : 0; break; + case NE: lhs = lhs != rhs ? 1 : 0; break; + case LAND: lhs = (lhs != 0) && (rhs != 0) ? 1 : 0; break; + case LOR: lhs = (lhs != 0) || (rhs != 0) ? 1 : 0; break; + + case '?': + /* XXX Handle this? */ + + default: + error(op, + "Unexpected operator " + op.getText()); + return 0; + + } + } + + /* + System.out.flush(); + (new Exception("expr returning " + lhs)).printStackTrace(); + System.err.flush(); + */ + // System.out.println("expr returning " + lhs); + + return lhs; + } + + private Token toWhitespace(Token tok) { + String text = tok.getText(); + int len = text.length(); + boolean cr = false; + int nls = 0; + + for (int i = 0; i < len; i++) { + char c = text.charAt(i); + + switch (c) { + case '\r': + cr = true; + nls++; + break; + case '\n': + if (cr) { + cr = false; + break; + } + /* fallthrough */ + case '\u2028': + case '\u2029': + case '\u000B': + case '\u000C': + case '\u0085': + cr = false; + nls++; + break; + } + } + + char[] cbuf = new char[nls]; + Arrays.fill(cbuf, '\n'); + return new Token(WHITESPACE, + tok.getLine(), tok.getColumn(), + new String(cbuf)); + } + + private final Token _token() + throws IOException, + LexerException { + + Token tok; + for (;;) { + if (!isActive()) { + /* Tell lexer to ignore warnings. */ + tok = source_token(); + /* Tell lexer to stop ignoring warnings. */ + switch (tok.getType()) { + case HASH: + case NL: + case EOF: + /* The preprocessor has to take action here. */ + break; + case WHITESPACE: + case COMMENT: + // Patch up to preserve whitespace. + /* XXX We might want to return tok here in C */ + return toWhitespace(tok); + default: + // Return NL to preserve whitespace. + return source_skipline(false); + } + } + else { + tok = source_token(); + } + + LEX: switch (tok.getType()) { + case EOF: + /* Pop the stacks. */ + return tok; + + case WHITESPACE: + case NL: + return tok; + + case COMMENT: + return tok; + + case '!': case '%': case '&': + case '(': case ')': case '*': + case '+': case ',': case '-': + case '/': case ':': case ';': + case '<': case '=': case '>': + case '?': case '[': case ']': + case '^': case '{': case '|': + case '}': case '~': case '.': + + // case '#': + + case AND_EQ: + case ARROW: + case CHARACTER: + case DEC: + case DIV_EQ: + case ELLIPSIS: + case EQ: + case GE: + case HEADER: /* Should only arise from include() */ + case INC: + case LAND: + case LE: + case LOR: + case LSH: + case LSH_EQ: + case SUB_EQ: + case MOD_EQ: + case MULT_EQ: + case NE: + case OR_EQ: + case PLUS_EQ: + case RANGE: + case RSH: + case RSH_EQ: + case STRING: + case XOR_EQ: + return tok; + + case INTEGER: + return tok; + + case IDENTIFIER: + Macro m = macros.get(tok.getText()); + if (m == null) + return tok; + if (source.isExpanding(m)) + return tok; + if (macro(m, tok)) + break; + return tok; + + case P_LINE: + if ((flags & FL_LINEMARKER) != 0) + return tok; + break; + + case ERROR: + return tok; + + default: + throw new InternalException("Bad token " + tok); + // break; + + case HASH: + tok = source_token_nonwhite(); + // (new Exception("here")).printStackTrace(); + switch (tok.getType()) { + case NL: + break LEX; /* Some code has #\n */ + case IDENTIFIER: + break; + default: + error(tok, + "Preprocessor directive not a word " + + tok.getText()); + return source_skipline(false); + } + Integer _ppcmd = ppcmds.get(tok.getText()); + if (_ppcmd == null) { + error(tok, + "Unknown preprocessor directive " + + tok.getText()); + return source_skipline(false); + } + int ppcmd = _ppcmd.intValue(); + + switch (ppcmd) { + + case PP_DEFINE: + if (!isActive()) + return source_skipline(false); + else + return define(); + // break; + + case PP_UNDEF: + if (!isActive()) + return source_skipline(false); + else + return undef(); + // break; + + case PP_INCLUDE: + if (!isActive()) + return source_skipline(false); + else + return include(); + // break; + + case PP_WARNING: + case PP_ERROR: + if (!isActive()) + return source_skipline(false); + else + error(tok, ppcmd == PP_ERROR); + break; + + case PP_IF: + push_state(); + if (!isActive()) { + return source_skipline(false); + } + expr_token = null; + states.peek().setActive(expr(0) != 0); + tok = expr_token(); /* unget */ + if (tok.getType() == NL) + return tok; + return source_skipline(true); + // break; + + case PP_ELIF: + State state = states.peek(); + if (false) { + /* Check for 'if' */ ; + } + else if (state.sawElse()) { + error(tok, + "#elif after #" + "else"); + return source_skipline(false); + } + else if (!state.isParentActive()) { + /* Nested in skipped 'if' */ + return source_skipline(false); + } + else if (state.isActive()) { + /* The 'if' part got executed. */ + state.setParentActive(false); + /* This is like # else # if but with + * only one # end. */ + state.setActive(false); + return source_skipline(false); + } + else { + expr_token = null; + state.setActive(expr(0) != 0); + tok = expr_token(); /* unget */ + if (tok.getType() == NL) + return tok; + return source_skipline(true); + } + // break; + + case PP_ELSE: + state = states.peek(); + if (false) + /* Check for 'if' */ ; + else if (state.sawElse()) { + error(tok, + "#" + "else after #" + "else"); + return source_skipline(false); + } + else { + state.setSawElse(); + state.setActive(! state.isActive()); + return source_skipline(true); + } + // break; + + case PP_IFDEF: + push_state(); + if (!isActive()) { + return source_skipline(false); + } + else { + tok = source_token_nonwhite(); + // System.out.println("ifdef " + tok); + if (tok.getType() != IDENTIFIER) { + error(tok, + "Expected identifier, not " + + tok.getText()); + return source_skipline(false); + } + else { + String text = tok.getText(); + boolean exists = + macros.containsKey(text); + states.peek().setActive(exists); + return source_skipline(true); + } + } + // break; + + case PP_IFNDEF: + push_state(); + if (!isActive()) { + return source_skipline(false); + } + else { + tok = source_token_nonwhite(); + if (tok.getType() != IDENTIFIER) { + error(tok, + "Expected identifier, not " + + tok.getText()); + return source_skipline(false); + } + else { + String text = tok.getText(); + boolean exists = + macros.containsKey(text); + states.peek().setActive(!exists); + return source_skipline(true); + } + } + // break; + + case PP_ENDIF: + pop_state(); + return source_skipline(true); + // break; + + case PP_LINE: + return source_skipline(false); + // break; + + case PP_PRAGMA: + return source_skipline(false); + // break; + + default: + /* Actual unknown directives are + * processed above. If we get here, + * we succeeded the map lookup but + * failed to handle it. Therefore, + * this is (unconditionally?) fatal. */ + // if (isActive()) /* XXX Could be warning. */ + throw new InternalException( + "Internal error: Unknown directive " + + tok); + // return source_skipline(false); + } + + + } + } + } + + private Token token_nonwhite() + throws IOException, + LexerException { + Token tok; + do { + tok = _token(); + } while (isWhite(tok)); + return tok; + } + + /** + * Returns the next preprocessor token. + * + * @see Token + * @throws LexerException if a preprocessing error occurs. + * @throws InternalException if an unexpected error condition arises. + */ + public Token token() + throws IOException, + LexerException { + Token tok = _token(); + if (DEBUG) + System.out.println("pp: Returning " + tok); + return tok; + } + +#set ($i = 1) /* First ppcmd is 1, not 0. */ +#set ($ppcmds = [ "define", "elif", "else", "endif", "error", "if", "ifdef", "ifndef", "include", "line", "pragma", "undef", "warning" ]) +#foreach ($ppcmd in $ppcmds) + private static final int PP_$ppcmd.toUpperCase() = $i; +#set ($i = $i + 1) +#end + + private static final Map<String,Integer> ppcmds = + new HashMap<String,Integer>(); + + static { +#foreach ($ppcmd in $ppcmds) + ppcmds.put("$ppcmd", Integer.valueOf(PP_$ppcmd.toUpperCase())); +#end + } + + + public String toString() { + StringBuilder buf = new StringBuilder(); + + Source s = getSource(); + while (s != null) { + buf.append(" -> ").append(String.valueOf(s)).append("\n"); + s = s.getParent(); + } + + Map<String,Macro> macros = getMacros(); + List<String> keys = new ArrayList<String>( + macros.keySet() + ); + Collections.sort(keys); + Iterator<String> mt = keys.iterator(); + while (mt.hasNext()) { + String key = mt.next(); + Macro macro = macros.get(key); + buf.append("#").append("macro ").append(macro).append("\n"); + } + + return buf.toString(); + } + +} diff --git a/src/java/org/anarres/cpp/PreprocessorListener.java b/src/java/org/anarres/cpp/PreprocessorListener.java new file mode 100644 index 0000000..84a105d --- /dev/null +++ b/src/java/org/anarres/cpp/PreprocessorListener.java @@ -0,0 +1,83 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.File; + +public class PreprocessorListener { + + private int errors; + private int warnings; + + public PreprocessorListener() { + clear(); + } + + public void clear() { + errors = 0; + warnings = 0; + } + + public int getErrors() { + return errors; + } + + public int getWarnings() { + return warnings; + } + + protected void print(String msg) { + System.err.println(msg); + } + + /** + * Handles a warning. + * + * The behaviour of this method is defined by the + * implementation. It may simply record the error message, or + * it may throw an exception. + */ + public void handleWarning(Source source, int line, int column, + String msg) + throws LexerException { + warnings++; + print(source.getName() + ":" + line + ":" + column + + ": warning: " + msg); + } + + /** + * Handles an error. + * + * The behaviour of this method is defined by the + * implementation. It may simply record the error message, or + * it may throw an exception. + */ + public void handleError(Source source, int line, int column, + String msg) + throws LexerException { + errors++; + print(source.getName() + ":" + line + ":" + column + + ": error: " + msg); + } + + public void handleSourceChange(Source source, String event) { + } + +} diff --git a/src/java/org/anarres/cpp/Source.java b/src/java/org/anarres/cpp/Source.java new file mode 100644 index 0000000..2999418 --- /dev/null +++ b/src/java/org/anarres/cpp/Source.java @@ -0,0 +1,226 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.PushbackReader; +import java.io.Reader; +import java.io.StringReader; + +import java.util.List; +import java.util.Iterator; + +import static org.anarres.cpp.Token.*; + +/** + * An input to the Preprocessor. + * + * Inputs may come from Files, Strings or other sources. The + * preprocessor maintains a stack of Sources. Operations such as + * file inclusion or token pasting will push a new source onto + * the Preprocessor stack. Sources pop from the stack when they + * are exhausted; this may be transparent or explicit. + * + * BUG: Error messages are not handled properly. + */ +public abstract class Source implements Iterable<Token> { + private Source parent; + private boolean autopop; + private PreprocessorListener listener; + + /* LineNumberReader */ + +/* + // We can't do this, since we would lose the LexerException + private class Itr implements Iterator { + private Token next = null; + private void advance() { + try { + if (next != null) + next = token(); + } + catch (IOException e) { + throw new UnsupportedOperationException( + "Failed to advance token iterator: " + + e.getMessage() + ); + } + } + public boolean hasNext() { + return next.getType() != EOF; + } + public Token next() { + advance(); + Token t = next; + next = null; + return t; + } + public void remove() { + throw new UnsupportedOperationException( + "Cannot remove tokens from a Source." + ); + } + } +*/ + + public Source() { + this.parent = null; + this.autopop = false; + } + + /* pp */ void setParent(Source parent, boolean autopop) { + this.parent = parent; + this.autopop = autopop; + } + + /* pp */ final Source getParent() { + return parent; + } + + /* pp */ void setListener(PreprocessorListener listener) { + this.listener = listener; + } + + /** + * Returns the File currently being lexed. + * + * If this Source is not a {@link FileLexerSource}, then + * it will ask the parent Source, and so forth recursively. + * If no Source on the stack is a FileLexerSource, returns null. + */ + /* pp */ File getFile() { + Source parent = getParent(); + while (parent != null) { + File file = parent.getFile(); + if (file != null) + return file; + parent = parent.getParent(); + } + return null; + } + + /* pp */ String getName() { + Source parent = getParent(); + while (parent != null) { + String name = parent.getName(); + if (name != null) + return name; + parent = parent.getParent(); + } + return null; + } + + public int getLine() { + Source parent = getParent(); + if (parent == null) + return 0; + return parent.getLine(); + } + + /* pp */ boolean isExpanding(Macro m) { + Source parent = getParent(); + if (parent != null) + return parent.isExpanding(m); + return false; + } + + /** + * Returns true if this Source should be transparently popped + * from the input stack. + * + * Examples of such sources are macro expansions. + */ + /* pp */ boolean isAutopop() { + return autopop; + } + + /* pp */ boolean isNumbered() { + return false; + } + + /** + * Returns the next Token parsed from this input stream. + * + * @see Token + */ + public abstract Token token() + throws IOException, + LexerException; + + public Iterator<Token> iterator() { + return new SourceIterator(this); + } + + /** + * Skips tokens until the end of line. + * + * @param white true if only whitespace is permitted on the + * remainder of the line. + * @return the NL token. + */ + public Token skipline(boolean white) + throws IOException, + LexerException { + for (;;) { + Token tok = token(); + switch (tok.getType()) { + case EOF: + /* There ought to be a newline before EOF. + * At least, in any skipline context. */ + /* XXX Are we sure about this? */ + warning(tok.getLine(), tok.getColumn(), + "No newline before end of file"); + return tok; + case NL: + /* This may contain one or more newlines. */ + return tok; + case COMMENT: + case WHITESPACE: + break; + default: + /* XXX Check white, if required. */ + if (white) + warning(tok.getLine(), tok.getColumn(), + "Unexpected nonwhite token"); + break; + } + } + } + + protected void error(int line, int column, String msg) + throws LexerException { + if (listener != null) + listener.handleError(this, line, column, msg); + else + throw new LexerException("No handler for error at " + line + ":" + column + ": " + msg); + } + + protected void warning(int line, int column, String msg) + throws LexerException { + if (listener != null) + listener.handleWarning(this, line, column, msg); + else + throw new LexerException("No handler for warning at " + line + ":" + column + ": " + msg); + } + +} diff --git a/src/java/org/anarres/cpp/SourceIterator.java b/src/java/org/anarres/cpp/SourceIterator.java new file mode 100644 index 0000000..ac2bc24 --- /dev/null +++ b/src/java/org/anarres/cpp/SourceIterator.java @@ -0,0 +1,94 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.IOException; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import static org.anarres.cpp.Token.*; + +/** + * An Iterator for {@link Source Sources}, + * returning {@link Token Tokens}. + */ +public class SourceIterator implements Iterator<Token> { + private Source source; + private Token tok; + + public SourceIterator(Source s) { + this.source = s; + this.tok = null; + } + + /** + * Rethrows IOException inside IllegalStateException. + */ + private void advance() { + try { + if (tok == null) + tok = source.token(); + } + catch (LexerException e) { + throw new IllegalStateException(e); + } + catch (IOException e) { + throw new IllegalStateException(e); + } + } + + /** + * Returns true if the enclosed Source has more tokens. + * + * The EOF token is never returned by the iterator. + * @throws IllegalStateException if the Source + * throws a LexerException or IOException + */ + public boolean hasNext() { + advance(); + return tok.getType() != EOF; + } + + /** + * Returns the next token from the enclosed Source. + * + * The EOF token is never returned by the iterator. + * @throws IllegalStateException if the Source + * throws a LexerException or IOException + */ + public Token next() { + if (!hasNext()) + throw new NoSuchElementException(); + Token t = this.tok; + this.tok = null; + return t; + } + + /** + * Not supported. + * + * @throws UnsupportedOperationException. + */ + public void remove() { + throw new UnsupportedOperationException(); + } +} + diff --git a/src/java/org/anarres/cpp/State.java b/src/java/org/anarres/cpp/State.java new file mode 100644 index 0000000..441e71e --- /dev/null +++ b/src/java/org/anarres/cpp/State.java @@ -0,0 +1,69 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +/* pp */ class State { + boolean parent; + boolean active; + boolean sawElse; + + /* pp */ State() { + this.parent = true; + this.active = true; + this.sawElse = false; + } + + /* pp */ State(State parent) { + this.parent = parent.isParentActive() && parent.isActive(); + this.active = true; + this.sawElse = false; + } + + /* Required for #elif */ + /* pp */ void setParentActive(boolean b) { + this.parent = b; + } + + /* pp */ boolean isParentActive() { + return parent; + } + + /* pp */ void setActive(boolean b) { + this.active = b; + } + + /* pp */ boolean isActive() { + return active; + } + + /* pp */ void setSawElse() { + sawElse = true; + } + + /* pp */ boolean sawElse() { + return sawElse; + } + + public String toString() { + return "parent=" + parent + + ", active=" + active + + ", sawelse=" + sawElse; + } +} diff --git a/src/java/org/anarres/cpp/StringLexerSource.java b/src/java/org/anarres/cpp/StringLexerSource.java new file mode 100644 index 0000000..7e7df75 --- /dev/null +++ b/src/java/org/anarres/cpp/StringLexerSource.java @@ -0,0 +1,64 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; + +import java.util.List; +import java.util.Iterator; + +import static org.anarres.cpp.Token.*; + +/** + * A Source for lexing a String. + * + * This class is used by token pasting, but can be used by user + * code. + */ +public class StringLexerSource extends LexerSource { + + /** + * Creates a new Source for lexing the given String. + * + * @param ppvalid true if preprocessor directives are to be + * honoured within the string. + */ + public StringLexerSource(String string, boolean ppvalid) + throws IOException { + super(new StringReader(string), ppvalid); + } + + /** + * Creates a new Source for lexing the given String. + * + * By default, preprocessor directives are not honoured within + * the string. + */ + public StringLexerSource(String string) + throws IOException { + this(string, false); + } + + public String toString() { + return "string literal"; + } +} diff --git a/src/java/org/anarres/cpp/Token.java b/src/java/org/anarres/cpp/Token.java new file mode 100644 index 0000000..e5c1319 --- /dev/null +++ b/src/java/org/anarres/cpp/Token.java @@ -0,0 +1,215 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +/** + * A Preprocessor token. + * + * @see Preprocessor + */ +public final class Token { + + // public static final int EOF = -1; + + private int type; + private int line; + private int column; + private Object value; + private String text; + + public Token(int type, int line, int column, + String text, Object value) { + this.type = type; + this.line = line; + this.column = column; + this.text = text; + this.value = value; + } + + public Token(int type, int line, int column, String text) { + this(type, line, column, text, null); + } + + /* pp */ Token(int type, String text, Object value) { + this(type, -1, -1, text, value); + } + + /* pp */ Token(int type, String text) { + this(type, text, null); + } + + /* pp */ Token(int type) { + this(type, texts[type]); + } + + /** + * Returns the semantic type of this token. + */ + public int getType() { + return type; + } + + /* pp */ void setLocation(int line, int column) { + this.line = line; + this.column = column; + } + + /** + * Returns the line at which this token started. + * + * Lines are numbered from zero. + */ + public int getLine() { + return line; + } + + /** + * Returns the column at which this token started. + * + * Columns are numbered from zero. + */ + public int getColumn() { + return column; + } + + /** + * Returns the original or generated text of this token. + * + * This is distinct from the semantic value of the token. + * + * @see #getValue() + */ + public String getText() { + return text; + } + + /** + * Returns the semantic value of this token. + * + * For strings, this is the parsed String. + * For integers, this is an Integer object. + * For other token types, as appropriate. + * + * @see #getText() + */ + public Object getValue() { + return value; + } + + /** + * Returns a description of this token, for debugging purposes. + */ + public String toString() { + StringBuilder buf = new StringBuilder(); + + buf.append('[').append(getTokenName(type)); + if (line != -1) { + buf.append('@').append(line); + if (column != -1) + buf.append(',').append(column); + } + buf.append("]:"); + if (text != null) + buf.append('"').append(text).append('"'); + else if (type > 3 && type < 256) + buf.append( (char)type ); + else + buf.append('<').append(type).append('>'); + if (value != null) + buf.append('=').append(value); + return buf.toString(); + } + + /** + * Returns the descriptive name of the given token type. + * + * This is mostly used for stringification and debugging. + */ + public static final String getTokenName(int type) { + if (type < 0) + return "Invalid" + type; + if (type >= names.length) + return "Invalid" + type; + if (names[type] == null) + return "Unknown" + type; + return names[type]; + } + +#set ($i = 257) +#set ($tokens = [ "AND_EQ", "ARROW", "CHARACTER", "COMMENT", "DEC", "DIV_EQ", "ELLIPSIS", "EOF", "EQ", "GE", "HASH", "HEADER", "IDENTIFIER", "INC", "INTEGER", "LAND", "LAND_EQ", "LE", "LITERAL", "LOR", "LOR_EQ", "LSH", "LSH_EQ", "MOD_EQ", "MULT_EQ", "NE", "NL", "OR_EQ", "PASTE", "PLUS_EQ", "RANGE", "RSH", "RSH_EQ", "STRING", "SUB_EQ", "WHITESPACE", "XOR_EQ", "M_ARG", "M_PASTE", "M_STRING", "P_LINE", "ERROR" ]) +#foreach ($token in $tokens) + /** The token type $token. */ + public static final int $token = $i; +#set ($i = $i + 1) +#end + /** + * The number of possible semantic token types. + * + * Please note that not all token types below 255 are used. + */ + public static final int _TOKENS = $i; + + /** The position-less space token. */ + /* pp */ static final Token space = new Token(WHITESPACE, -1, -1, " "); + + private static final String[] names = new String[_TOKENS]; + private static final String[] texts = new String[_TOKENS]; + static { + for (int i = 0; i < 255; i++) { + texts[i] = String.valueOf(new char[] { (char)i }); + names[i] = texts[i]; + } + + texts[AND_EQ] = "&="; + texts[ARROW] = "->"; + texts[DEC] = "--"; + texts[DIV_EQ] = "/="; + texts[ELLIPSIS] = "..."; + texts[EQ] = "=="; + texts[GE] = ">="; + texts[HASH] = "#"; + texts[INC] = "++"; + texts[LAND] = "&&"; + texts[LAND_EQ] = "&&="; + texts[LE] = "<="; + texts[LOR] = "||"; + texts[LOR_EQ] = "||="; + texts[LSH] = "<<"; + texts[LSH_EQ] = "<<="; + texts[MOD_EQ] = "%="; + texts[MULT_EQ] = "*="; + texts[NE] = "!="; + texts[NL] = "\n"; + texts[OR_EQ] = "|="; + /* We have to split the two hashes or Velocity eats them. */ + texts[PASTE] = "#" + "#"; + texts[PLUS_EQ] = "+="; + texts[RANGE] = ".."; + texts[RSH] = ">>"; + texts[RSH_EQ] = ">>="; + texts[SUB_EQ] = "-="; + texts[XOR_EQ] = "^="; + +#foreach ($token in $tokens) + names[$token] = "$token"; +#end + } + +} diff --git a/src/java/org/anarres/cpp/TokenSnifferSource.java b/src/java/org/anarres/cpp/TokenSnifferSource.java new file mode 100644 index 0000000..55b53d7 --- /dev/null +++ b/src/java/org/anarres/cpp/TokenSnifferSource.java @@ -0,0 +1,56 @@ +/* + * Anarres C Preprocessor + * Copyright (C) 2007 Shevek + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package org.anarres.cpp; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.PushbackReader; +import java.io.Reader; +import java.io.StringReader; + +import java.util.ArrayList; +import java.util.List; +import java.util.Iterator; + +import static org.anarres.cpp.Token.*; + +@Deprecated +/* pp */ class TokenSnifferSource extends Source { + private List<Token> target; + + /* pp */ TokenSnifferSource(List<Token> target) { + this.target = target; + } + + public Token token() + throws IOException, + LexerException { + Token tok = getParent().token(); + if (tok.getType() != EOF) + target.add(tok); + return tok; + } + + public String toString() { + return getParent().toString(); + } +} |