From bdc6c852f418c3e042aa41469d84544e5f60a526 Mon Sep 17 00:00:00 2001 From: Shevek Date: Fri, 27 Dec 2013 05:49:13 -0800 Subject: Version bump to 1.4.0-SNAPSHOT. Rewrite build system to use gradle. Clean up source for the new generation. --- src/main/java/org/anarres/cpp/Preprocessor.java | 2016 +++++++++++++++++++++++ 1 file changed, 2016 insertions(+) create mode 100644 src/main/java/org/anarres/cpp/Preprocessor.java (limited to 'src/main/java/org/anarres/cpp/Preprocessor.java') diff --git a/src/main/java/org/anarres/cpp/Preprocessor.java b/src/main/java/org/anarres/cpp/Preprocessor.java new file mode 100644 index 0000000..35480b2 --- /dev/null +++ b/src/main/java/org/anarres/cpp/Preprocessor.java @@ -0,0 +1,2016 @@ +/* + * Anarres C Preprocessor + * Copyright (c) 2007-2008, Shevek + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package org.anarres.cpp; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Stack; + +import javax.annotation.Nonnull; +import static org.anarres.cpp.Token.*; +import static org.anarres.cpp.PreprocessorCommand.*; + +/** + * A C Preprocessor. + * The Preprocessor outputs a token stream which does not need + * re-lexing for C or C++. Alternatively, the output text may be + * reconstructed by concatenating the {@link Token#getText() text} + * values of the returned {@link Token Tokens}. (See + * {@link CppReader}, which does this.) + */ +/* + Source file name and line number information is conveyed by lines of the form + + # linenum filename flags + + These are called linemarkers. They are inserted as needed into + the output (but never within a string or character constant). They + mean that the following line originated in file filename at line + linenum. filename will never contain any non-printing characters; + they are replaced with octal escape sequences. + + After the file name comes zero or more flags, which are `1', `2', + `3', or `4'. If there are multiple flags, spaces separate them. Here + is what the flags mean: + + `1' + This indicates the start of a new file. + `2' + This indicates returning to a file (after having included another + file). + `3' + This indicates that the following text comes from a system header + file, so certain warnings should be suppressed. + `4' + This indicates that the following text should be treated as being + wrapped in an implicit extern "C" block. + */ +public class Preprocessor implements Closeable { + + private static final Source INTERNAL = new Source() { + @Override + public Token token() + throws IOException, + LexerException { + throw new LexerException("Cannot read from " + getName()); + } + + @Override + public String getPath() { + return ""; + } + + @Override + public String getName() { + return "internal data"; + } + }; + private static final Macro __LINE__ = new Macro(INTERNAL, "__LINE__"); + private static final Macro __FILE__ = new Macro(INTERNAL, "__FILE__"); + private static final Macro __COUNTER__ = new Macro(INTERNAL, "__COUNTER__"); + + private List inputs; + + /* The fundamental engine. */ + private Map macros; + private Stack states; + private Source source; + + /* Miscellaneous support. */ + private int counter; + + /* Support junk to make it work like cpp */ + private List quoteincludepath; /* -iquote */ + + private List sysincludepath; /* -I */ + + private List frameworkspath; + private Set features; + private Set warnings; + private VirtualFileSystem filesystem; + private PreprocessorListener listener; + + public Preprocessor() { + this.inputs = new ArrayList(); + + this.macros = new HashMap(); + macros.put(__LINE__.getName(), __LINE__); + macros.put(__FILE__.getName(), __FILE__); + macros.put(__COUNTER__.getName(), __COUNTER__); + this.states = new Stack(); + states.push(new State()); + this.source = null; + + this.counter = 0; + + this.quoteincludepath = new ArrayList(); + this.sysincludepath = new ArrayList(); + this.frameworkspath = new ArrayList(); + this.features = EnumSet.noneOf(Feature.class); + this.warnings = EnumSet.noneOf(Warning.class); + this.filesystem = new JavaFileSystem(); + this.listener = null; + } + + public Preprocessor(Source initial) { + this(); + addInput(initial); + } + + /** Equivalent to + * 'new Preprocessor(new {@link FileLexerSource}(file))' + */ + public Preprocessor(File file) + throws IOException { + this(new FileLexerSource(file)); + } + + /** + * Sets the VirtualFileSystem used by this Preprocessor. + */ + public void setFileSystem(VirtualFileSystem filesystem) { + this.filesystem = filesystem; + } + + /** + * Returns the VirtualFileSystem used by this Preprocessor. + */ + public VirtualFileSystem getFileSystem() { + return filesystem; + } + + /** + * Sets the PreprocessorListener which handles events for + * this Preprocessor. + * + * The listener is notified of warnings, errors and source + * changes, amongst other things. + */ + public void setListener(PreprocessorListener listener) { + this.listener = listener; + Source s = source; + while (s != null) { + // s.setListener(listener); + s.init(this); + s = s.getParent(); + } + } + + /** + * Returns the PreprocessorListener which handles events for + * this Preprocessor. + */ + public PreprocessorListener getListener() { + return listener; + } + + /** + * Returns the feature-set for this Preprocessor. + * + * This set may be freely modified by user code. + */ + public Set getFeatures() { + return features; + } + + /** + * Adds a feature to the feature-set of this Preprocessor. + */ + public void addFeature(Feature f) { + features.add(f); + } + + /** + * Adds features to the feature-set of this Preprocessor. + */ + public void addFeatures(Collection f) { + features.addAll(f); + } + + /** + * Returns true if the given feature is in + * the feature-set of this Preprocessor. + */ + public boolean getFeature(Feature f) { + return features.contains(f); + } + + /** + * Returns the warning-set for this Preprocessor. + * + * This set may be freely modified by user code. + */ + public Set getWarnings() { + return warnings; + } + + /** + * Adds a warning to the warning-set of this Preprocessor. + */ + public void addWarning(Warning w) { + warnings.add(w); + } + + /** + * Adds warnings to the warning-set of this Preprocessor. + */ + public void addWarnings(Collection w) { + warnings.addAll(w); + } + + /** + * Returns true if the given warning is in + * the warning-set of this Preprocessor. + */ + public boolean getWarning(Warning w) { + return warnings.contains(w); + } + + /** + * Adds input for the Preprocessor. + * + * Inputs are processed in the order in which they are added. + */ + public void addInput(Source source) { + source.init(this); + inputs.add(source); + } + + /** + * Adds input for the Preprocessor. + * + * @see #addInput(Source) + */ + public void addInput(File file) + throws IOException { + addInput(new FileLexerSource(file)); + } + + /** + * Handles an error. + * + * If a PreprocessorListener is installed, it receives the + * error. Otherwise, an exception is thrown. + */ + protected void error(int line, int column, String msg) + throws LexerException { + if (listener != null) + listener.handleError(source, line, column, msg); + else + throw new LexerException("Error at " + line + ":" + column + ": " + msg); + } + + /** + * Handles an error. + * + * If a PreprocessorListener is installed, it receives the + * error. Otherwise, an exception is thrown. + * + * @see #error(int, int, String) + */ + protected void error(Token tok, String msg) + throws LexerException { + error(tok.getLine(), tok.getColumn(), msg); + } + + /** + * Handles a warning. + * + * If a PreprocessorListener is installed, it receives the + * warning. Otherwise, an exception is thrown. + */ + protected void warning(int line, int column, String msg) + throws LexerException { + if (warnings.contains(Warning.ERROR)) + error(line, column, msg); + else if (listener != null) + listener.handleWarning(source, line, column, msg); + else + throw new LexerException("Warning at " + line + ":" + column + ": " + msg); + } + + /** + * Handles a warning. + * + * If a PreprocessorListener is installed, it receives the + * warning. Otherwise, an exception is thrown. + * + * @see #warning(int, int, String) + */ + protected void warning(Token tok, String msg) + throws LexerException { + warning(tok.getLine(), tok.getColumn(), msg); + } + + /** + * Adds a Macro to this Preprocessor. + * + * The given {@link Macro} object encapsulates both the name + * and the expansion. + */ + public void addMacro(Macro m) throws LexerException { + // System.out.println("Macro " + m); + String name = m.getName(); + /* Already handled as a source error in macro(). */ + if ("defined".equals(name)) + throw new LexerException("Cannot redefine name 'defined'"); + macros.put(m.getName(), m); + } + + /** + * Defines the given name as a macro. + * + * The String value is lexed into a token stream, which is + * used as the macro expansion. + */ + public void addMacro(String name, String value) + throws LexerException { + try { + Macro m = new Macro(name); + StringLexerSource s = new StringLexerSource(value); + for (;;) { + Token tok = s.token(); + if (tok.getType() == EOF) + break; + m.addToken(tok); + } + addMacro(m); + } catch (IOException e) { + throw new LexerException(e); + } + } + + /** + * Defines the given name as a macro, with the value 1. + * + * This is a convnience method, and is equivalent to + * addMacro(name, "1"). + */ + public void addMacro(String name) + throws LexerException { + addMacro(name, "1"); + } + + /** + * Sets the user include path used by this Preprocessor. + */ + /* Note for future: Create an IncludeHandler? */ + public void setQuoteIncludePath(List path) { + this.quoteincludepath = path; + } + + /** + * Returns the user include-path of this Preprocessor. + * + * This list may be freely modified by user code. + */ + public List getQuoteIncludePath() { + return quoteincludepath; + } + + /** + * Sets the system include path used by this Preprocessor. + */ + /* Note for future: Create an IncludeHandler? */ + public void setSystemIncludePath(List path) { + this.sysincludepath = path; + } + + /** + * Returns the system include-path of this Preprocessor. + * + * This list may be freely modified by user code. + */ + public List getSystemIncludePath() { + return sysincludepath; + } + + /** + * Sets the Objective-C frameworks path used by this Preprocessor. + */ + /* Note for future: Create an IncludeHandler? */ + public void setFrameworksPath(List path) { + this.frameworkspath = path; + } + + /** + * Returns the Objective-C frameworks path used by this + * Preprocessor. + * + * This list may be freely modified by user code. + */ + public List getFrameworksPath() { + return frameworkspath; + } + + /** + * Returns the Map of Macros parsed during the run of this + * Preprocessor. + */ + public Map getMacros() { + return macros; + } + + /** + * Returns the named macro. + * + * While you can modify the returned object, unexpected things + * might happen if you do. + */ + public Macro getMacro(String name) { + return macros.get(name); + } + + /* States */ + private void push_state() { + State top = states.peek(); + states.push(new State(top)); + } + + private void pop_state() + throws LexerException { + State s = states.pop(); + if (states.isEmpty()) { + error(0, 0, "#" + "endif without #" + "if"); + states.push(s); + } + } + + private boolean isActive() { + State state = states.peek(); + return state.isParentActive() && state.isActive(); + } + + + /* Sources */ + /** + * Returns the top Source on the input stack. + * + * @see Source + * @see #push_source(Source,boolean) + * @see #pop_source() + */ + protected Source getSource() { + return source; + } + + /** + * Pushes a Source onto the input stack. + * + * @see #getSource() + * @see #pop_source() + */ + protected void push_source(Source source, boolean autopop) { + source.init(this); + source.setParent(this.source, autopop); + // source.setListener(listener); + if (listener != null) + listener.handleSourceChange(this.source, "suspend"); + this.source = source; + if (listener != null) + listener.handleSourceChange(this.source, "push"); + } + + /** + * Pops a Source from the input stack. + * + * @see #getSource() + * @see #push_source(Source,boolean) + */ + protected void pop_source() + throws IOException { + if (listener != null) + listener.handleSourceChange(this.source, "pop"); + Source s = this.source; + this.source = s.getParent(); + /* Always a noop unless called externally. */ + s.close(); + if (listener != null && this.source != null) + listener.handleSourceChange(this.source, "resume"); + } + + + /* Source tokens */ + private Token source_token; + + /* XXX Make this include the NL, and make all cpp directives eat + * their own NL. */ + private Token line_token(int line, String name, String extra) { + StringBuilder buf = new StringBuilder(); + buf.append("#line ").append(line) + .append(" \""); + /* XXX This call to escape(name) is correct but ugly. */ + MacroTokenSource.escape(buf, name); + buf.append("\"").append(extra).append("\n"); + return new Token(P_LINE, line, 0, buf.toString(), null); + } + + private Token source_token() + throws IOException, + LexerException { + if (source_token != null) { + Token tok = source_token; + source_token = null; + if (getFeature(Feature.DEBUG)) + System.err.println("Returning unget token " + tok); + return tok; + } + + for (;;) { + Source s = getSource(); + if (s == null) { + if (inputs.isEmpty()) + return new Token(EOF); + Source t = inputs.remove(0); + push_source(t, true); + if (getFeature(Feature.LINEMARKERS)) + return line_token(t.getLine(), t.getName(), " 1"); + continue; + } + Token tok = s.token(); + /* XXX Refactor with skipline() */ + if (tok.getType() == EOF && s.isAutopop()) { + // System.out.println("Autopop " + s); + pop_source(); + Source t = getSource(); + if (getFeature(Feature.LINEMARKERS) + && s.isNumbered() + && t != null) { + /* We actually want 'did the nested source + * contain a newline token', which isNumbered() + * approximates. This is not perfect, but works. */ + return line_token(t.getLine() + 1, t.getName(), " 2"); + } + continue; + } + if (getFeature(Feature.DEBUG)) + System.err.println("Returning fresh token " + tok); + return tok; + } + } + + private void source_untoken(Token tok) { + if (this.source_token != null) + throw new IllegalStateException("Cannot return two tokens"); + this.source_token = tok; + } + + private boolean isWhite(Token tok) { + int type = tok.getType(); + return (type == WHITESPACE) + || (type == CCOMMENT) + || (type == CPPCOMMENT); + } + + private Token source_token_nonwhite() + throws IOException, + LexerException { + Token tok; + do { + tok = source_token(); + } while (isWhite(tok)); + return tok; + } + + /** + * Returns an NL or an EOF token. + * + * The metadata on the token will be correct, which is better + * than generating a new one. + * + * This method can, as of recent patches, return a P_LINE token. + */ + private Token source_skipline(boolean white) + throws IOException, + LexerException { + // (new Exception("skipping line")).printStackTrace(System.out); + Source s = getSource(); + Token tok = s.skipline(white); + /* XXX Refactor with source_token() */ + if (tok.getType() == EOF && s.isAutopop()) { + // System.out.println("Autopop " + s); + pop_source(); + Source t = getSource(); + if (getFeature(Feature.LINEMARKERS) + && s.isNumbered() + && t != null) { + /* We actually want 'did the nested source + * contain a newline token', which isNumbered() + * approximates. This is not perfect, but works. */ + return line_token(t.getLine() + 1, t.getName(), " 2"); + } + } + return tok; + } + + /* processes and expands a macro. */ + private boolean macro(Macro m, Token orig) + throws IOException, + LexerException { + Token tok; + List args; + + // System.out.println("pp: expanding " + m); + if (m.isFunctionLike()) { + OPEN: + for (;;) { + tok = source_token(); + // System.out.println("pp: open: token is " + tok); + switch (tok.getType()) { + case WHITESPACE: /* XXX Really? */ + + case CCOMMENT: + case CPPCOMMENT: + case NL: + break; /* continue */ + + case '(': + break OPEN; + default: + source_untoken(tok); + return false; + } + } + + // tok = expanded_token_nonwhite(); + tok = source_token_nonwhite(); + + /* We either have, or we should have args. + * This deals elegantly with the case that we have + * one empty arg. */ + if (tok.getType() != ')' || m.getArgs() > 0) { + args = new ArrayList(); + + Argument arg = new Argument(); + int depth = 0; + boolean space = false; + + ARGS: + for (;;) { + // System.out.println("pp: arg: token is " + tok); + switch (tok.getType()) { + case EOF: + error(tok, "EOF in macro args"); + return false; + + case ',': + if (depth == 0) { + if (m.isVariadic() + && /* We are building the last arg. */ args.size() == m.getArgs() - 1) { + /* Just add the comma. */ + arg.addToken(tok); + } else { + args.add(arg); + arg = new Argument(); + } + } else { + arg.addToken(tok); + } + space = false; + break; + case ')': + if (depth == 0) { + args.add(arg); + break ARGS; + } else { + depth--; + arg.addToken(tok); + } + space = false; + break; + case '(': + depth++; + arg.addToken(tok); + space = false; + break; + + case WHITESPACE: + case CCOMMENT: + case CPPCOMMENT: + /* Avoid duplicating spaces. */ + space = true; + break; + + default: + /* Do not put space on the beginning of + * an argument token. */ + if (space && !arg.isEmpty()) + arg.addToken(Token.space); + arg.addToken(tok); + space = false; + break; + + } + // tok = expanded_token(); + tok = source_token(); + } + /* space may still be true here, thus trailing space + * is stripped from arguments. */ + + if (args.size() != m.getArgs()) { + error(tok, + "macro " + m.getName() + + " has " + m.getArgs() + " parameters " + + "but given " + args.size() + " args"); + /* We could replay the arg tokens, but I + * note that GNU cpp does exactly what we do, + * i.e. output the macro name and chew the args. + */ + return false; + } + + /* + for (Argument a : args) + a.expand(this); + */ + for (int i = 0; i < args.size(); i++) { + args.get(i).expand(this); + } + + // System.out.println("Macro " + m + " args " + args); + } else { + /* nargs == 0 and we (correctly) got () */ + args = null; + } + + } else { + /* Macro without args. */ + args = null; + } + + if (m == __LINE__) { + push_source(new FixedTokenSource( + new Token[]{new Token(NUMBER, + orig.getLine(), orig.getColumn(), + String.valueOf(orig.getLine()), + new NumericValue(10, "" + orig.getLine()))} + ), true); + } else if (m == __FILE__) { + StringBuilder buf = new StringBuilder("\""); + String name = getSource().getName(); + if (name == null) + name = ""; + for (int i = 0; i < name.length(); i++) { + char c = name.charAt(i); + switch (c) { + case '\\': + buf.append("\\\\"); + break; + case '"': + buf.append("\\\""); + break; + default: + buf.append(c); + break; + } + } + buf.append("\""); + String text = buf.toString(); + push_source(new FixedTokenSource( + new Token[]{new Token(STRING, + orig.getLine(), orig.getColumn(), + text, text)} + ), true); + } else if (m == __COUNTER__) { + /* This could equivalently have been done by adding + * a special Macro subclass which overrides getTokens(). */ + int value = this.counter++; + push_source(new FixedTokenSource( + new Token[]{new Token(NUMBER, + orig.getLine(), orig.getColumn(), + String.valueOf(value), + new NumericValue(10, "" + value))} + ), true); + } else { + push_source(new MacroTokenSource(m, args), true); + } + + return true; + } + + /** + * Expands an argument. + */ + /* I'd rather this were done lazily, but doing so breaks spec. */ + /* pp */ List expand(List arg) + throws IOException, + LexerException { + List expansion = new ArrayList(); + boolean space = false; + + push_source(new FixedTokenSource(arg), false); + + EXPANSION: + for (;;) { + Token tok = expanded_token(); + switch (tok.getType()) { + case EOF: + break EXPANSION; + + case WHITESPACE: + case CCOMMENT: + case CPPCOMMENT: + space = true; + break; + + default: + if (space && !expansion.isEmpty()) + expansion.add(Token.space); + expansion.add(tok); + space = false; + break; + } + } + + pop_source(); + + return expansion; + } + + /* processes a #define directive */ + private Token define() + throws IOException, + LexerException { + Token tok = source_token_nonwhite(); + if (tok.getType() != IDENTIFIER) { + error(tok, "Expected identifier"); + return source_skipline(false); + } + /* if predefined */ + + String name = tok.getText(); + if ("defined".equals(name)) { + error(tok, "Cannot redefine name 'defined'"); + return source_skipline(false); + } + + Macro m = new Macro(getSource(), name); + List args; + + tok = source_token(); + if (tok.getType() == '(') { + tok = source_token_nonwhite(); + if (tok.getType() != ')') { + args = new ArrayList(); + ARGS: + for (;;) { + switch (tok.getType()) { + case IDENTIFIER: + args.add(tok.getText()); + break; + case NL: + case EOF: + error(tok, + "Unterminated macro parameter list"); + return tok; + default: + error(tok, + "error in macro parameters: " + + tok.getText()); + return source_skipline(false); + } + tok = source_token_nonwhite(); + switch (tok.getType()) { + case ',': + break; + case ELLIPSIS: + tok = source_token_nonwhite(); + if (tok.getType() != ')') + error(tok, + "ellipsis must be on last argument"); + m.setVariadic(true); + break ARGS; + case ')': + break ARGS; + + case NL: + case EOF: + /* Do not skip line. */ + error(tok, + "Unterminated macro parameters"); + return tok; + default: + error(tok, + "Bad token in macro parameters: " + + tok.getText()); + return source_skipline(false); + } + tok = source_token_nonwhite(); + } + } else { + assert tok.getType() == ')' : "Expected ')'"; + args = Collections.emptyList(); + } + + m.setArgs(args); + } else { + /* For searching. */ + args = Collections.emptyList(); + source_untoken(tok); + } + + /* Get an expansion for the macro, using indexOf. */ + boolean space = false; + boolean paste = false; + int idx; + + /* Ensure no space at start. */ + tok = source_token_nonwhite(); + EXPANSION: + for (;;) { + switch (tok.getType()) { + case EOF: + break EXPANSION; + case NL: + break EXPANSION; + + case CCOMMENT: + case CPPCOMMENT: + /* XXX This is where we implement GNU's cpp -CC. */ + // break; + case WHITESPACE: + if (!paste) + space = true; + break; + + /* Paste. */ + case PASTE: + space = false; + paste = true; + m.addPaste(new Token(M_PASTE, + tok.getLine(), tok.getColumn(), + "#" + "#", null)); + break; + + /* Stringify. */ + case '#': + if (space) + m.addToken(Token.space); + space = false; + Token la = source_token_nonwhite(); + if (la.getType() == IDENTIFIER + && ((idx = args.indexOf(la.getText())) != -1)) { + m.addToken(new Token(M_STRING, + la.getLine(), la.getColumn(), + "#" + la.getText(), + Integer.valueOf(idx))); + } else { + m.addToken(tok); + /* Allow for special processing. */ + source_untoken(la); + } + break; + + case IDENTIFIER: + if (space) + m.addToken(Token.space); + space = false; + paste = false; + idx = args.indexOf(tok.getText()); + if (idx == -1) + m.addToken(tok); + else + m.addToken(new Token(M_ARG, + tok.getLine(), tok.getColumn(), + tok.getText(), + Integer.valueOf(idx))); + break; + + default: + if (space) + m.addToken(Token.space); + space = false; + paste = false; + m.addToken(tok); + break; + } + tok = source_token(); + } + + if (getFeature(Feature.DEBUG)) + System.err.println("Defined macro " + m); + addMacro(m); + + return tok; /* NL or EOF. */ + + } + + private Token undef() + throws IOException, + LexerException { + Token tok = source_token_nonwhite(); + if (tok.getType() != IDENTIFIER) { + error(tok, + "Expected identifier, not " + tok.getText()); + if (tok.getType() == NL || tok.getType() == EOF) + return tok; + } else { + Macro m = macros.get(tok.getText()); + if (m != null) { + /* XXX error if predefined */ + macros.remove(m.getName()); + } + } + return source_skipline(true); + } + + /** + * Attempts to include the given file. + * + * User code may override this method to implement a virtual + * file system. + */ + private boolean include(VirtualFile file) + throws IOException, + LexerException { + // System.out.println("Try to include " + ((File)file).getAbsolutePath()); + if (!file.isFile()) + return false; + if (getFeature(Feature.DEBUG)) + System.err.println("pp: including " + file); + push_source(file.getSource(), true); + return true; + } + + /** + * Includes a file from an include path, by name. + */ + private boolean include(Iterable path, String name) + throws IOException, + LexerException { + for (String dir : path) { + VirtualFile file = filesystem.getFile(dir, name); + if (include(file)) + return true; + } + return false; + } + + /** + * Handles an include directive. + */ + private void include(String parent, int line, + String name, boolean quoted) + throws IOException, + LexerException { + VirtualFile pdir = null; + if (quoted) { + VirtualFile pfile = filesystem.getFile(parent); + pdir = pfile.getParentFile(); + VirtualFile ifile = pdir.getChildFile(name); + if (include(ifile)) + return; + if (include(quoteincludepath, name)) + return; + } + + if (include(sysincludepath, name)) + return; + + StringBuilder buf = new StringBuilder(); + buf.append("File not found: ").append(name); + buf.append(" in"); + if (quoted) { + buf.append(" .").append('(').append(pdir).append(')'); + for (String dir : quoteincludepath) + buf.append(" ").append(dir); + } + for (String dir : sysincludepath) + buf.append(" ").append(dir); + error(line, 0, buf.toString()); + } + + private Token include(boolean next) + throws IOException, + LexerException { + LexerSource lexer = (LexerSource) source; + try { + lexer.setInclude(true); + Token tok = token_nonwhite(); + + String name; + boolean quoted; + + if (tok.getType() == STRING) { + /* XXX Use the original text, not the value. + * Backslashes must not be treated as escapes here. */ + StringBuilder buf = new StringBuilder((String) tok.getValue()); + HEADER: + for (;;) { + tok = token_nonwhite(); + switch (tok.getType()) { + case STRING: + buf.append((String) tok.getValue()); + break; + case NL: + case EOF: + break HEADER; + default: + warning(tok, + "Unexpected token on #" + "include line"); + return source_skipline(false); + } + } + name = buf.toString(); + quoted = true; + } else if (tok.getType() == HEADER) { + name = (String) tok.getValue(); + quoted = false; + tok = source_skipline(true); + } else { + error(tok, + "Expected string or header, not " + tok.getText()); + switch (tok.getType()) { + case NL: + case EOF: + return tok; + default: + /* Only if not a NL or EOF already. */ + return source_skipline(false); + } + } + + /* Do the inclusion. */ + include(source.getPath(), tok.getLine(), name, quoted); + + /* 'tok' is the 'nl' after the include. We use it after the + * #line directive. */ + if (getFeature(Feature.LINEMARKERS)) + return line_token(1, source.getName(), " 1"); + return tok; + } finally { + lexer.setInclude(false); + } + } + + protected void pragma(Token name, List value) + throws IOException, + LexerException { + warning(name, "Unknown #" + "pragma: " + name.getText()); + } + + private Token pragma() + throws IOException, + LexerException { + Token name; + + NAME: + for (;;) { + Token tok = token(); + switch (tok.getType()) { + case EOF: + /* There ought to be a newline before EOF. + * At least, in any skipline context. */ + /* XXX Are we sure about this? */ + warning(tok, + "End of file in #" + "pragma"); + return tok; + case NL: + /* This may contain one or more newlines. */ + warning(tok, + "Empty #" + "pragma"); + return tok; + case CCOMMENT: + case CPPCOMMENT: + case WHITESPACE: + continue NAME; + case IDENTIFIER: + name = tok; + break NAME; + default: + return source_skipline(false); + } + } + + Token tok; + List value = new ArrayList(); + VALUE: + for (;;) { + tok = token(); + switch (tok.getType()) { + case EOF: + /* There ought to be a newline before EOF. + * At least, in any skipline context. */ + /* XXX Are we sure about this? */ + warning(tok, + "End of file in #" + "pragma"); + break VALUE; + case NL: + /* This may contain one or more newlines. */ + break VALUE; + case CCOMMENT: + case CPPCOMMENT: + break; + case WHITESPACE: + value.add(tok); + break; + default: + value.add(tok); + break; + } + } + + pragma(name, value); + + return tok; /* The NL. */ + + } + + /* For #error and #warning. */ + private void error(Token pptok, boolean is_error) + throws IOException, + LexerException { + StringBuilder buf = new StringBuilder(); + buf.append('#').append(pptok.getText()).append(' '); + /* Peculiar construction to ditch first whitespace. */ + Token tok = source_token_nonwhite(); + ERROR: + for (;;) { + switch (tok.getType()) { + case NL: + case EOF: + break ERROR; + default: + buf.append(tok.getText()); + break; + } + tok = source_token(); + } + if (is_error) + error(pptok, buf.toString()); + else + warning(pptok, buf.toString()); + } + + /* This bypasses token() for #elif expressions. + * If we don't do this, then isActive() == false + * causes token() to simply chew the entire input line. */ + private Token expanded_token() + throws IOException, + LexerException { + for (;;) { + Token tok = source_token(); + // System.out.println("Source token is " + tok); + if (tok.getType() == IDENTIFIER) { + Macro m = macros.get(tok.getText()); + if (m == null) + return tok; + if (source.isExpanding(m)) + return tok; + if (macro(m, tok)) + continue; + } + return tok; + } + } + + private Token expanded_token_nonwhite() + throws IOException, + LexerException { + Token tok; + do { + tok = expanded_token(); + // System.out.println("expanded token is " + tok); + } while (isWhite(tok)); + return tok; + } + + private Token expr_token = null; + + private Token expr_token() + throws IOException, + LexerException { + Token tok = expr_token; + + if (tok != null) { + // System.out.println("ungetting"); + expr_token = null; + } else { + tok = expanded_token_nonwhite(); + // System.out.println("expt is " + tok); + + if (tok.getType() == IDENTIFIER + && tok.getText().equals("defined")) { + Token la = source_token_nonwhite(); + boolean paren = false; + if (la.getType() == '(') { + paren = true; + la = source_token_nonwhite(); + } + + // System.out.println("Core token is " + la); + if (la.getType() != IDENTIFIER) { + error(la, + "defined() needs identifier, not " + + la.getText()); + tok = new Token(NUMBER, + la.getLine(), la.getColumn(), + "0", new NumericValue(10, "0")); + } else if (macros.containsKey(la.getText())) { + // System.out.println("Found macro"); + tok = new Token(NUMBER, + la.getLine(), la.getColumn(), + "1", new NumericValue(10, "1")); + } else { + // System.out.println("Not found macro"); + tok = new Token(NUMBER, + la.getLine(), la.getColumn(), + "0", new NumericValue(10, "0")); + } + + if (paren) { + la = source_token_nonwhite(); + if (la.getType() != ')') { + expr_untoken(la); + error(la, "Missing ) in defined()"); + } + } + } + } + + // System.out.println("expr_token returns " + tok); + return tok; + } + + private void expr_untoken(Token tok) + throws LexerException { + if (expr_token != null) + throw new InternalException( + "Cannot unget two expression tokens." + ); + expr_token = tok; + } + + private int expr_priority(Token op) { + switch (op.getType()) { + case '/': + return 11; + case '%': + return 11; + case '*': + return 11; + case '+': + return 10; + case '-': + return 10; + case LSH: + return 9; + case RSH: + return 9; + case '<': + return 8; + case '>': + return 8; + case LE: + return 8; + case GE: + return 8; + case EQ: + return 7; + case NE: + return 7; + case '&': + return 6; + case '^': + return 5; + case '|': + return 4; + case LAND: + return 3; + case LOR: + return 2; + case '?': + return 1; + default: + // System.out.println("Unrecognised operator " + op); + return 0; + } + } + + private long expr(int priority) + throws IOException, + LexerException { + /* + System.out.flush(); + (new Exception("expr(" + priority + ") called")).printStackTrace(); + System.err.flush(); + */ + + Token tok = expr_token(); + long lhs, rhs; + + // System.out.println("Expr lhs token is " + tok); + switch (tok.getType()) { + case '(': + lhs = expr(0); + tok = expr_token(); + if (tok.getType() != ')') { + expr_untoken(tok); + error(tok, "missing ) in expression"); + return 0; + } + break; + + case '~': + lhs = ~expr(11); + break; + case '!': + lhs = expr(11) == 0 ? 1 : 0; + break; + case '-': + lhs = -expr(11); + break; + case NUMBER: + NumericValue value = (NumericValue) tok.getValue(); + lhs = value.longValue(); + break; + case CHARACTER: + lhs = (long) ((Character) tok.getValue()).charValue(); + break; + case IDENTIFIER: + if (warnings.contains(Warning.UNDEF)) + warning(tok, "Undefined token '" + tok.getText() + + "' encountered in conditional."); + lhs = 0; + break; + + default: + expr_untoken(tok); + error(tok, + "Bad token in expression: " + tok.getText()); + return 0; + } + + EXPR: + for (;;) { + // System.out.println("expr: lhs is " + lhs + ", pri = " + priority); + Token op = expr_token(); + int pri = expr_priority(op); /* 0 if not a binop. */ + + if (pri == 0 || priority >= pri) { + expr_untoken(op); + break EXPR; + } + rhs = expr(pri); + // System.out.println("rhs token is " + rhs); + switch (op.getType()) { + case '/': + if (rhs == 0) { + error(op, "Division by zero"); + lhs = 0; + } else { + lhs = lhs / rhs; + } + break; + case '%': + if (rhs == 0) { + error(op, "Modulus by zero"); + lhs = 0; + } else { + lhs = lhs % rhs; + } + break; + case '*': + lhs = lhs * rhs; + break; + case '+': + lhs = lhs + rhs; + break; + case '-': + lhs = lhs - rhs; + break; + case '<': + lhs = lhs < rhs ? 1 : 0; + break; + case '>': + lhs = lhs > rhs ? 1 : 0; + break; + case '&': + lhs = lhs & rhs; + break; + case '^': + lhs = lhs ^ rhs; + break; + case '|': + lhs = lhs | rhs; + break; + + case LSH: + lhs = lhs << rhs; + break; + case RSH: + lhs = lhs >> rhs; + break; + case LE: + lhs = lhs <= rhs ? 1 : 0; + break; + case GE: + lhs = lhs >= rhs ? 1 : 0; + break; + case EQ: + lhs = lhs == rhs ? 1 : 0; + break; + case NE: + lhs = lhs != rhs ? 1 : 0; + break; + case LAND: + lhs = (lhs != 0) && (rhs != 0) ? 1 : 0; + break; + case LOR: + lhs = (lhs != 0) || (rhs != 0) ? 1 : 0; + break; + + case '?': + /* XXX Handle this? */ + + default: + error(op, + "Unexpected operator " + op.getText()); + return 0; + + } + } + + /* + System.out.flush(); + (new Exception("expr returning " + lhs)).printStackTrace(); + System.err.flush(); + */ + // System.out.println("expr returning " + lhs); + return lhs; + } + + private Token toWhitespace(Token tok) { + String text = tok.getText(); + int len = text.length(); + boolean cr = false; + int nls = 0; + + for (int i = 0; i < len; i++) { + char c = text.charAt(i); + + switch (c) { + case '\r': + cr = true; + nls++; + break; + case '\n': + if (cr) { + cr = false; + break; + } + /* fallthrough */ + case '\u2028': + case '\u2029': + case '\u000B': + case '\u000C': + case '\u0085': + cr = false; + nls++; + break; + } + } + + char[] cbuf = new char[nls]; + Arrays.fill(cbuf, '\n'); + return new Token(WHITESPACE, + tok.getLine(), tok.getColumn(), + new String(cbuf)); + } + + @Nonnull + private Token _token() + throws IOException, + LexerException { + + for (;;) { + Token tok; + if (!isActive()) { + try { + /* XXX Tell lexer to ignore warnings. */ + source.setActive(false); + tok = source_token(); + } finally { + /* XXX Tell lexer to stop ignoring warnings. */ + source.setActive(true); + } + switch (tok.getType()) { + case HASH: + case NL: + case EOF: + /* The preprocessor has to take action here. */ + break; + case WHITESPACE: + return tok; + case CCOMMENT: + case CPPCOMMENT: + // Patch up to preserve whitespace. + if (getFeature(Feature.KEEPALLCOMMENTS)) + return tok; + if (!isActive()) + return toWhitespace(tok); + if (getFeature(Feature.KEEPCOMMENTS)) + return tok; + return toWhitespace(tok); + default: + // Return NL to preserve whitespace. + /* XXX This might lose a comment. */ + return source_skipline(false); + } + } else { + tok = source_token(); + } + + LEX: + switch (tok.getType()) { + case EOF: + /* Pop the stacks. */ + return tok; + + case WHITESPACE: + case NL: + return tok; + + case CCOMMENT: + case CPPCOMMENT: + return tok; + + case '!': + case '%': + case '&': + case '(': + case ')': + case '*': + case '+': + case ',': + case '-': + case '/': + case ':': + case ';': + case '<': + case '=': + case '>': + case '?': + case '[': + case ']': + case '^': + case '{': + case '|': + case '}': + case '~': + case '.': + + /* From Olivier Chafik for Objective C? */ + case '@': + /* The one remaining ASCII, might as well. */ + case '`': + + // case '#': + case AND_EQ: + case ARROW: + case CHARACTER: + case DEC: + case DIV_EQ: + case ELLIPSIS: + case EQ: + case GE: + case HEADER: /* Should only arise from include() */ + + case INC: + case LAND: + case LE: + case LOR: + case LSH: + case LSH_EQ: + case SUB_EQ: + case MOD_EQ: + case MULT_EQ: + case NE: + case OR_EQ: + case PLUS_EQ: + case RANGE: + case RSH: + case RSH_EQ: + case STRING: + case XOR_EQ: + return tok; + + case NUMBER: + return tok; + + case IDENTIFIER: + Macro m = macros.get(tok.getText()); + if (m == null) + return tok; + if (source.isExpanding(m)) + return tok; + if (macro(m, tok)) + break; + return tok; + + case P_LINE: + if (getFeature(Feature.LINEMARKERS)) + return tok; + break; + + case INVALID: + if (getFeature(Feature.CSYNTAX)) + error(tok, String.valueOf(tok.getValue())); + return tok; + + default: + throw new InternalException("Bad token " + tok); + // break; + + case HASH: + tok = source_token_nonwhite(); + // (new Exception("here")).printStackTrace(); + switch (tok.getType()) { + case NL: + break LEX; /* Some code has #\n */ + + case IDENTIFIER: + break; + default: + error(tok, + "Preprocessor directive not a word " + + tok.getText()); + return source_skipline(false); + } + PreprocessorCommand ppcmd = PreprocessorCommand.forText(tok.getText()); + if (ppcmd == null) { + error(tok, + "Unknown preprocessor directive " + + tok.getText()); + return source_skipline(false); + } + + PP: + switch (ppcmd) { + + case PP_DEFINE: + if (!isActive()) + return source_skipline(false); + else + return define(); + // break; + + case PP_UNDEF: + if (!isActive()) + return source_skipline(false); + else + return undef(); + // break; + + case PP_INCLUDE: + if (!isActive()) + return source_skipline(false); + else + return include(false); + // break; + case PP_INCLUDE_NEXT: + if (!isActive()) + return source_skipline(false); + if (!getFeature(Feature.INCLUDENEXT)) { + error(tok, + "Directive include_next not enabled" + ); + return source_skipline(false); + } + return include(true); + // break; + + case PP_WARNING: + case PP_ERROR: + if (!isActive()) + return source_skipline(false); + else + error(tok, ppcmd == PP_ERROR); + break; + + case PP_IF: + push_state(); + if (!isActive()) { + return source_skipline(false); + } + expr_token = null; + states.peek().setActive(expr(0) != 0); + tok = expr_token(); /* unget */ + + if (tok.getType() == NL) + return tok; + return source_skipline(true); + // break; + + case PP_ELIF: + State state = states.peek(); + if (false) { + /* Check for 'if' */; + } else if (state.sawElse()) { + error(tok, + "#elif after #" + "else"); + return source_skipline(false); + } else if (!state.isParentActive()) { + /* Nested in skipped 'if' */ + return source_skipline(false); + } else if (state.isActive()) { + /* The 'if' part got executed. */ + state.setParentActive(false); + /* This is like # else # if but with + * only one # end. */ + state.setActive(false); + return source_skipline(false); + } else { + expr_token = null; + state.setActive(expr(0) != 0); + tok = expr_token(); /* unget */ + + if (tok.getType() == NL) + return tok; + return source_skipline(true); + } + // break; + + case PP_ELSE: + state = states.peek(); + if (false) + /* Check for 'if' */ ; else if (state.sawElse()) { + error(tok, + "#" + "else after #" + "else"); + return source_skipline(false); + } else { + state.setSawElse(); + state.setActive(!state.isActive()); + return source_skipline(warnings.contains(Warning.ENDIF_LABELS)); + } + // break; + + case PP_IFDEF: + push_state(); + if (!isActive()) { + return source_skipline(false); + } else { + tok = source_token_nonwhite(); + // System.out.println("ifdef " + tok); + if (tok.getType() != IDENTIFIER) { + error(tok, + "Expected identifier, not " + + tok.getText()); + return source_skipline(false); + } else { + String text = tok.getText(); + boolean exists + = macros.containsKey(text); + states.peek().setActive(exists); + return source_skipline(true); + } + } + // break; + + case PP_IFNDEF: + push_state(); + if (!isActive()) { + return source_skipline(false); + } else { + tok = source_token_nonwhite(); + if (tok.getType() != IDENTIFIER) { + error(tok, + "Expected identifier, not " + + tok.getText()); + return source_skipline(false); + } else { + String text = tok.getText(); + boolean exists + = macros.containsKey(text); + states.peek().setActive(!exists); + return source_skipline(true); + } + } + // break; + + case PP_ENDIF: + pop_state(); + return source_skipline(warnings.contains(Warning.ENDIF_LABELS)); + // break; + + case PP_LINE: + return source_skipline(false); + // break; + + case PP_PRAGMA: + if (!isActive()) + return source_skipline(false); + return pragma(); + // break; + + default: + /* Actual unknown directives are + * processed above. If we get here, + * we succeeded the map lookup but + * failed to handle it. Therefore, + * this is (unconditionally?) fatal. */ + // if (isActive()) /* XXX Could be warning. */ + throw new InternalException( + "Internal error: Unknown directive " + + tok); + // return source_skipline(false); + } + + } + } + } + + private Token token_nonwhite() + throws IOException, + LexerException { + Token tok; + do { + tok = _token(); + } while (isWhite(tok)); + return tok; + } + + /** + * Returns the next preprocessor token. + * + * @see Token + * @throws LexerException if a preprocessing error occurs. + * @throws InternalException if an unexpected error condition arises. + */ + @Nonnull + public Token token() + throws IOException, + LexerException { + Token tok = _token(); + if (getFeature(Feature.DEBUG)) + System.err.println("pp: Returning " + tok); + return tok; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(); + + Source s = getSource(); + while (s != null) { + buf.append(" -> ").append(String.valueOf(s)).append("\n"); + s = s.getParent(); + } + + Map macros = getMacros(); + List keys = new ArrayList( + macros.keySet() + ); + Collections.sort(keys); + Iterator mt = keys.iterator(); + while (mt.hasNext()) { + String key = mt.next(); + Macro macro = macros.get(key); + buf.append("#").append("macro ").append(macro).append("\n"); + } + + return buf.toString(); + } + + @Override + public void close() + throws IOException { + { + Source s = source; + while (s != null) { + s.close(); + s = s.getParent(); + } + } + for (Source s : inputs) { + s.close(); + } + } + +} -- cgit v1.2.3