diff options
37 files changed, 406 insertions, 122 deletions
@@ -1,4 +1,64 @@ -build -.*.swp +# Compiled source # +################### +*.com +*.class +*.dll +*.exe +*.o +*.so + +# Packages # +############ +# it's better to unpack these files and commit the raw source +# git has its own built in compression methods +*.7z +*.dmg +*.gz +*.iso +*.rar +*.tar +*.zip + +# Logs and databases # +###################### +*.log + +# OS generated files # +###################### +.DS_Store* +ehthumbs.db +Icon? +Thumbs.db + +# Editor Files # +################ +*~ +*.swp + +# Gradle Files # +################ .gradle + +# Build output directies +/target +*/target +/build +*/build + +# IntelliJ specific files/directories +out +.idea +*.ipr +*.iws +*.iml +atlassian-ide-plugin.xml + +# Eclipse specific files/directories +.classpath +.project +.settings +.metadata + +# NetBeans specific files/directories +.nbattrs .nb-gradle diff --git a/src/main/java/com/jogamp/gluegen/jcpp/Argument.java b/src/main/java/com/jogamp/gluegen/jcpp/Argument.java index bccddcb..da66fc3 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/Argument.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/Argument.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/ChrootFileSystem.java b/src/main/java/com/jogamp/gluegen/jcpp/ChrootFileSystem.java index f3806fa..6baefa6 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/ChrootFileSystem.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/ChrootFileSystem.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/CppReader.java b/src/main/java/com/jogamp/gluegen/jcpp/CppReader.java index 82ef2fe..367194c 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/CppReader.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/CppReader.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -113,7 +113,10 @@ public class CppReader extends Reader implements Closeable { } return true; } catch (LexerException e) { - throw new IOException(String.valueOf(e), e); + // new IOException(String, Throwable) is since 1.6 + IOException _e = new IOException(String.valueOf(e)); + _e.initCause(e); + throw _e; } } @@ -125,6 +128,7 @@ public class CppReader extends Reader implements Closeable { return token.charAt(idx++); } + @Override /* XXX Very slow and inefficient. */ public int read(char cbuf[], int off, int len) throws IOException { diff --git a/src/main/java/com/jogamp/gluegen/jcpp/CppTask.java b/src/main/java/com/jogamp/gluegen/jcpp/CppTask.java index e3be324..da311f9 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/CppTask.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/CppTask.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,8 +25,6 @@ import java.util.Enumeration; import java.util.List; import org.apache.tools.ant.BuildException; import org.apache.tools.ant.taskdefs.Copy; -import org.apache.tools.ant.types.FilterSet; -import org.apache.tools.ant.types.FilterSetCollection; import org.apache.tools.ant.types.Path; /** @@ -117,6 +115,11 @@ public class CppTask extends Copy { } */ private void preprocess(File input, File output) throws Exception { + if (input == null) + throw new BuildException("Input not specified"); + if (output == null) + throw new BuildException("Output not specified"); + Preprocessor cpp = new Preprocessor(); cpp.setListener(listener); for (Macro macro : macros) @@ -135,10 +138,6 @@ public class CppTask extends Copy { } FileWriter writer = null; try { - if (input == null) - throw new BuildException("Input not specified"); - if (output == null) - throw new BuildException("Output not specified"); cpp.addInput(input); writer = new FileWriter(output); for (;;) { @@ -181,18 +180,19 @@ public class CppTask extends Copy { try { log("Copying " + fromFile + " to " + toFile, verbosity); - FilterSetCollection executionFilters - = new FilterSetCollection(); - if (filtering) { - executionFilters - .addFilterSet(getProject().getGlobalFilterSet()); - } - for (Enumeration filterEnum = getFilterSets().elements(); - filterEnum.hasMoreElements();) { - executionFilters - .addFilterSet((FilterSet) filterEnum.nextElement()); - } - + /* + FilterSetCollection executionFilters + = new FilterSetCollection(); + if (filtering) { + executionFilters + .addFilterSet(getProject().getGlobalFilterSet()); + } + for (Enumeration filterEnum = getFilterSets().elements(); + filterEnum.hasMoreElements();) { + executionFilters + .addFilterSet((FilterSet) filterEnum.nextElement()); + } + */ File srcFile = new File(fromFile); File dstFile = new File(toFile); preprocess(srcFile, dstFile); diff --git a/src/main/java/com/jogamp/gluegen/jcpp/DefaultPreprocessorListener.java b/src/main/java/com/jogamp/gluegen/jcpp/DefaultPreprocessorListener.java index aeff3c8..a3f3dd5 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/DefaultPreprocessorListener.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/DefaultPreprocessorListener.java @@ -2,7 +2,7 @@ package com.jogamp.gluegen.jcpp; /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/Feature.java b/src/main/java/com/jogamp/gluegen/jcpp/Feature.java index e2feefc..86202fc 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/Feature.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/Feature.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ public enum Feature { LINEMARKERS, /** Reports tokens of type INVALID as errors. */ CSYNTAX, - /** Preserves comments in the lexed output. */ + /** Preserves comments in the lexed output. Like cpp -C */ KEEPCOMMENTS, /** Preserves comments in the lexed output, even when inactive. */ KEEPALLCOMMENTS, diff --git a/src/main/java/com/jogamp/gluegen/jcpp/FileLexerSource.java b/src/main/java/com/jogamp/gluegen/jcpp/FileLexerSource.java index 6d7e4c5..01fdbd1 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/FileLexerSource.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/FileLexerSource.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,8 +18,10 @@ package com.jogamp.gluegen.jcpp; import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; +import java.nio.charset.Charset; import javax.annotation.Nonnull; /** @@ -29,7 +31,7 @@ import javax.annotation.Nonnull; * * @see Source */ -public class FileLexerSource extends LexerSource { +public class FileLexerSource extends InputLexerSource { private final String path; private final File file; @@ -39,29 +41,38 @@ public class FileLexerSource extends LexerSource { * * Preprocessor directives are honoured within the file. */ - public FileLexerSource(@Nonnull File file, String path) + public FileLexerSource(@Nonnull File file, @Nonnull Charset charset, @Nonnull String path) throws IOException { - super( - new BufferedReader( - new FileReader( - file - ) - ), - true - ); - + super(new FileInputStream(file), charset); this.file = file; this.path = path; } + public FileLexerSource(@Nonnull File file, @Nonnull String path) + throws IOException { + this(file, Charset.defaultCharset(), path); + } + + public FileLexerSource(@Nonnull File file, @Nonnull Charset charset) + throws IOException { + this(file, charset, file.getPath()); + } + + @Deprecated public FileLexerSource(@Nonnull File file) throws IOException { - this(file, file.getPath()); + this(file, Charset.defaultCharset()); + } + + public FileLexerSource(@Nonnull String path, @Nonnull Charset charset) + throws IOException { + this(new File(path), charset, path); } + @Deprecated public FileLexerSource(@Nonnull String path) throws IOException { - this(new File(path), path); + this(path, Charset.defaultCharset()); } @Nonnull diff --git a/src/main/java/com/jogamp/gluegen/jcpp/FixedTokenSource.java b/src/main/java/com/jogamp/gluegen/jcpp/FixedTokenSource.java index d084932..b954bda 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/FixedTokenSource.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/FixedTokenSource.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/InputLexerSource.java b/src/main/java/com/jogamp/gluegen/jcpp/InputLexerSource.java index 00cb01a..f4bd640 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/InputLexerSource.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/InputLexerSource.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,14 @@ */ package com.jogamp.gluegen.jcpp; -import java.io.BufferedReader; -import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.Charset; +import javax.annotation.Nonnull; /** - * A {@link Source} which lexes a file. + * A {@link Source} which lexes an {@link InputStream}. * * The input is buffered. * @@ -30,21 +31,26 @@ import java.io.InputStreamReader; */ public class InputLexerSource extends LexerSource { + @Deprecated + public InputLexerSource(@Nonnull InputStream input) { + this(input, Charset.defaultCharset()); + } + /** * Creates a new Source for lexing the given Reader. * * Preprocessor directives are honoured within the file. */ - public InputLexerSource(InputStream input) - throws IOException { - super( - new BufferedReader( - new InputStreamReader( - input - ) - ), - true - ); + public InputLexerSource(@Nonnull InputStream input, Charset charset) { + this(new InputStreamReader(input, charset)); + } + + public InputLexerSource(@Nonnull Reader input, boolean ppvalid) { + super(input, true); + } + + public InputLexerSource(@Nonnull Reader input) { + this(input, true); } @Override diff --git a/src/main/java/com/jogamp/gluegen/jcpp/InternalException.java b/src/main/java/com/jogamp/gluegen/jcpp/InternalException.java index 72abaf9..c404a62 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/InternalException.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/InternalException.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/JavaFileSystem.java b/src/main/java/com/jogamp/gluegen/jcpp/JavaFileSystem.java index f62ba53..a7acd36 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/JavaFileSystem.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/JavaFileSystem.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/JoinReader.java b/src/main/java/com/jogamp/gluegen/jcpp/JoinReader.java index e44a7e6..e2bba26 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/JoinReader.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/JoinReader.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/LexerException.java b/src/main/java/com/jogamp/gluegen/jcpp/LexerException.java index 6a9bf5b..f590b92 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/LexerException.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/LexerException.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/LexerSource.java b/src/main/java/com/jogamp/gluegen/jcpp/LexerSource.java index 798e0a9..29c231b 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/LexerSource.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/LexerSource.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ */ package com.jogamp.gluegen.jcpp; +import java.io.BufferedReader; import java.io.IOException; import java.io.Reader; @@ -26,6 +27,13 @@ import static com.jogamp.gluegen.jcpp.Token.*; /** Does not handle digraphs. */ public class LexerSource extends Source { + @Nonnull + protected static BufferedReader toBufferedReader(@Nonnull Reader r) { + if (r instanceof BufferedReader) + return (BufferedReader) r; + return new BufferedReader(r); + } + private static final boolean DEBUG = false; private JoinReader reader; @@ -70,11 +78,25 @@ public class LexerSource extends Source { this.reader.init(pp, this); } + /** + * Returns the line number of the last read character in this source. + * + * Lines are numbered from 1. + * + * @return the line number of the last read character in this source. + */ @Override public int getLine() { return line; } + /** + * Returns the column number of the last read character in this source. + * + * Columns are numbered from 0. + * + * @return the column number of the last read character in this source. + */ @Override public int getColumn() { return column; @@ -293,6 +315,14 @@ public class LexerSource extends Source { return new Token(CPPCOMMENT, text.toString()); } + /** + * Lexes an escaped character, appends the lexed escape sequence to 'text' and returns the parsed character value. + * + * @param text The buffer to which the literal escape sequence is appended. + * @return The new parsed character value. + * @throws IOException if it goes badly wrong. + * @throws LexerException if it goes wrong. + */ private int escape(StringBuilder text) throws IOException, LexerException { @@ -398,7 +428,7 @@ public class LexerSource extends Source { int e = read(); if (e != '\'') { // error("Illegal character constant"); - /* We consume up to the next ' or the rest of the line. */ + /* We consume up to the next ' or the rest of the line. */ for (;;) { if (isLineSeparator(e)) { unread(e); @@ -515,8 +545,7 @@ public class LexerSource extends Source { flags |= NumericValue.F_DOUBLE; text.append((char) d); d = read(); - } - else if (Character.isUnicodeIdentifierPart(d)) { + } else if (Character.isUnicodeIdentifierPart(d)) { String reason = "Invalid suffix \"" + (char) d + "\" on numeric constant"; // We've encountered something initially identified as a number. // Read in the rest of this token as an identifer but return it as an invalid. @@ -542,7 +571,7 @@ public class LexerSource extends Source { LexerException { StringBuilder part = new StringBuilder(); int d = read(); - if (sign && d == '-') { + if (sign && (d == '+' || d == '-')) { text.append((char) d); part.append((char) d); d = read(); @@ -630,9 +659,9 @@ public class LexerSource extends Source { /** * Section 6.4.4.1 of C99 - * + * * (Not pasted here, but says that the initial negation is a separate token.) - * + * * Section 6.4.4.2 of C99 * * A floating constant has a significand part that may be followed @@ -715,8 +744,7 @@ public class LexerSource extends Source { text.append((char) c); for (;;) { d = read(); - if (ppvalid && isLineSeparator(d)) /* XXX Ugly. */ - + if (ppvalid && isLineSeparator(d)) /* XXX Ugly. */ break; if (Character.isWhitespace(d)) text.append((char) d); @@ -952,7 +980,14 @@ public class LexerSource extends Source { } else if (Character.isJavaIdentifierStart(c)) { tok = identifier(c); } else { - tok = new Token(c); + String text = TokenType.getTokenText(c); + if (text == null) { + if ((c >>> 16) == 0) // Character.isBmpCodePoint() is new in 1.7 + text = Character.toString((char) c); + else + text = new String(Character.toChars(c)); + } + tok = new Token(c, text); } } diff --git a/src/main/java/com/jogamp/gluegen/jcpp/Macro.java b/src/main/java/com/jogamp/gluegen/jcpp/Macro.java index 3029252..765d37a 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/Macro.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/Macro.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/MacroTokenSource.java b/src/main/java/com/jogamp/gluegen/jcpp/MacroTokenSource.java index fbb2428..18839c1 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/MacroTokenSource.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/MacroTokenSource.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Iterator; import java.util.List; +import javax.annotation.Nonnegative; import javax.annotation.Nonnull; import static com.jogamp.gluegen.jcpp.Token.*; @@ -28,6 +29,7 @@ import static com.jogamp.gluegen.jcpp.Token.*; * in any inactive context. */ /* pp */ class MacroTokenSource extends Source { + private static final Logger LOG = LoggerFactory.getLogger(MacroTokenSource.class); private final Macro macro; private final Iterator<Token> tokens; /* Pointer into the macro. */ @@ -98,18 +100,34 @@ import static com.jogamp.gluegen.jcpp.Token.*; str.toString(), buf.toString()); } + /** + * Returns true if the given argumentIndex is the last argument of a variadic macro. + * + * @param argumentIndex The index of the argument to inspect. + * @return true if the given argumentIndex is the last argument of a variadic macro. + */ + private boolean isVariadicArgument(@Nonnegative int argumentIndex) { + if (!macro.isVariadic()) + return false; + return argumentIndex == args.size() - 1; + } /* At this point, we have consumed the first M_PASTE. * @see Macro#addPaste(Token) */ private void paste(@Nonnull Token ptok) throws IOException, LexerException { + // List<Token> out = new ArrayList<Token>(); StringBuilder buf = new StringBuilder(); // Token err = null; /* We know here that arg is null or expired, * since we cannot paste an expanded arg. */ int count = 2; + // While I hate auxiliary booleans, this does actually seem to be the simplest solution, + // as it avoids duplicating all the logic around hasNext() in case COMMA. + boolean comma = false; + TOKEN: for (int i = 0; i < count; i++) { if (!tokens.hasNext()) { /* XXX This one really should throw. */ @@ -129,16 +147,29 @@ import static com.jogamp.gluegen.jcpp.Token.*; break; case M_ARG: int idx = ((Integer) tok.getValue()).intValue(); - concat(buf, args.get(idx)); + Argument arg = args.get(idx); + if (comma && isVariadicArgument(idx) && arg.isEmpty()) { + // Ugly way to strip the comma. + buf.setLength(buf.length() - 1); + } else { + concat(buf, arg); + } break; /* XXX Test this. */ case CCOMMENT: case CPPCOMMENT: + // TODO: In cpp, -CC keeps these comments too, + // but turns all C++ comments into C comments. break; + case ',': + comma = true; + buf.append(tok.getText()); + continue TOKEN; default: buf.append(tok.getText()); break; } + comma = false; } /* Push and re-lex. */ diff --git a/src/main/java/com/jogamp/gluegen/jcpp/NumericValue.java b/src/main/java/com/jogamp/gluegen/jcpp/NumericValue.java index 8e79fd3..7707c09 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/NumericValue.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/NumericValue.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,7 +41,7 @@ public class NumericValue extends Number { private String exponent; private int flags; - public NumericValue(int base, String integer) { + public NumericValue(@Nonnegative int base, @Nonnull String integer) { this.base = base; this.integer = integer; } @@ -61,7 +61,7 @@ public class NumericValue extends Number { return fraction; } - /* pp */ void setFractionalPart(String fraction) { + /* pp */ void setFractionalPart(@Nonnull String fraction) { this.fraction = fraction; } @@ -75,7 +75,7 @@ public class NumericValue extends Number { return exponent; } - /* pp */ void setExponent(int expbase, String exponent) { + /* pp */ void setExponent(@Nonnegative int expbase, @Nonnull String exponent) { this.expbase = expbase; this.exponent = exponent; } @@ -110,6 +110,10 @@ public class NumericValue extends Number { return new BigDecimal(unscaled, scale); } + // We could construct a heuristic for when an 'int' is large enough. + // private static final int S_MAXLEN_LONG = String.valueOf(Long.MAX_VALUE).length(); + // private static final int S_MAXLEN_INT = String.valueOf(Integer.MAX_VALUE).length(); + @Nonnull public Number toJavaLangNumber() { int flags = getFlags(); @@ -125,8 +129,15 @@ public class NumericValue extends Number { return doubleValue(); // .1 is a double in Java. else if (getExponent() != null) return doubleValue(); - else - return intValue(); + else { + // This is an attempt to avoid overflowing on over-long integers. + // However, now we just overflow on over-long longs. + // We should really use BigInteger. + long value = longValue(); + if (value <= Integer.MAX_VALUE && value >= Integer.MIN_VALUE) + return (int) value; + return value; + } } private int exponentValue() { @@ -135,7 +146,8 @@ public class NumericValue extends Number { @Override public int intValue() { - int v = integer.isEmpty() ? 0 : Integer.parseInt(integer, base); + // String.isEmpty() is since 1.6 + int v = integer.length() == 0 ? 0 : Integer.parseInt(integer, base); if (expbase == 2) v = v << exponentValue(); else if (expbase != 0) @@ -145,7 +157,8 @@ public class NumericValue extends Number { @Override public long longValue() { - long v = integer.isEmpty() ? 0 : Long.parseLong(integer, base); + // String.isEmpty() is since 1.6 + long v = integer.length() == 0 ? 0 : Long.parseLong(integer, base); if (expbase == 2) v = v << exponentValue(); else if (expbase != 0) diff --git a/src/main/java/com/jogamp/gluegen/jcpp/Preprocessor.java b/src/main/java/com/jogamp/gluegen/jcpp/Preprocessor.java index 3208b0e..3427467 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/Preprocessor.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/Preprocessor.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -355,7 +355,8 @@ public class Preprocessor implements Closeable { * * The given {@link Macro} object encapsulates both the name * and the expansion. - * @throws IOException + * + * @throws LexerException if the definition fails or is otherwise illegal. */ public void addMacro(@Nonnull final Macro m) throws LexerException, IOException { // System.out.println("Macro " + m); @@ -372,6 +373,8 @@ public class Preprocessor implements Closeable { * * The String value is lexed into a token stream, which is * used as the macro expansion. + * + * @throws LexerException if the definition fails or is otherwise illegal. */ public void addMacro(@Nonnull final String name, @Nonnull final String value) throws LexerException { @@ -399,6 +402,8 @@ public class Preprocessor implements Closeable { * * This is a convnience method, and is equivalent to * <code>addMacro(name, "1")</code>. + * + * @throws LexerException if the definition fails or is otherwise illegal. */ public void addMacro(@Nonnull final String name) throws LexerException { @@ -463,6 +468,8 @@ public class Preprocessor implements Closeable { /** * Returns the Map of Macros parsed during the run of this * Preprocessor. + * + * @return The {@link Map} of macros currently defined. */ @Nonnull public Map<String, Macro> getMacros() { @@ -497,9 +504,11 @@ public class Preprocessor implements Closeable { * * While you can modify the returned object, unexpected things * might happen if you do. + * + * @return the Macro object, or null if not found. */ @CheckForNull - public Macro getMacro(final String name) { + public Macro getMacro(@Nonnull String name) { return macros.get(name); } @@ -543,6 +552,8 @@ public class Preprocessor implements Closeable { * @see Source * @see #push_source(Source,boolean) * @see #pop_source() + * + * @return the top Source on the input stack. */ // @CheckForNull public Source getSource() { @@ -552,6 +563,8 @@ public class Preprocessor implements Closeable { /** * Pushes a Source onto the input stack. * + * @param source the new Source to push onto the top of the input stack. + * @param autopop if true, the Source is automatically removed from the input stack at EOF. * @see #getSource() * @see #pop_source() */ @@ -571,6 +584,9 @@ public class Preprocessor implements Closeable { * * @see #getSource() * @see #push_source(Source,boolean) + * + * @param linemarker TODO: currently ignored, might be a bug? + * @throws IOException if an I/O error occurs. */ @CheckForNull protected Token pop_source(final boolean linemarker) @@ -590,11 +606,8 @@ public class Preprocessor implements Closeable { && t != null) { /* We actually want 'did the nested source * contain a newline token', which isNumbered() - * approximates. This is not perfect, but works. - * FIXME: Removed the '+ 1', since all lines were off by one. - * This solves this case, but I don't know _why_ this was here in the first place. - */ - return line_token(t.getLine() /* SEE ABOVE: + 1 */, t.getName(), " 2"); + * approximates. This is not perfect, but works. */ + return line_token(t.getLine(), t.getName(), " 2"); } return null; @@ -1149,10 +1162,13 @@ public class Preprocessor implements Closeable { * * User code may override this method to implement a virtual * file system. + * + * @param file The VirtualFile to attempt to include. + * @return true if the file was successfully included, false otherwise. + * @throws IOException if an I/O error occurs. */ - protected boolean include(@Nonnull final VirtualFile file) - throws IOException, - LexerException { + protected boolean include(@Nonnull VirtualFile file) + throws IOException { // System.out.println("Try to include " + ((File)file).getAbsolutePath()); if (!file.isFile()) return false; @@ -1164,13 +1180,17 @@ public class Preprocessor implements Closeable { } /** - * Includes a file from an include path, by name. + * Attempts to include a file from an include path, by name. + * + * @param path The list of virtual directories to search for the given name. + * @param name The name of the file to attempt to include. + * @return true if the file was successfully included, false otherwise. + * @throws IOException if an I/O error occurs. */ - protected boolean include(@Nonnull final Iterable<String> path, @Nonnull final String name) - throws IOException, - LexerException { - for (final String dir : path) { - final VirtualFile file = getFileSystem().getFile(dir, name); + protected boolean include(@Nonnull Iterable<String> path, @Nonnull String name) + throws IOException { + for (String dir : path) { + VirtualFile file = getFileSystem().getFile(dir, name); if (include(file)) return true; } @@ -1179,6 +1199,9 @@ public class Preprocessor implements Closeable { /** * Handles an include directive. + * + * @throws IOException if an I/O error occurs. + * @throws LexerException if the include fails, and the error handler is fatal. */ private void include( @CheckForNull final String parent, final int line, @@ -1330,7 +1353,7 @@ public class Preprocessor implements Closeable { NAME: for (;;) { - final Token tok = token(); + Token tok = source_token(); switch (tok.getType()) { case EOF: /* There ought to be a newline before EOF. @@ -1352,6 +1375,8 @@ public class Preprocessor implements Closeable { name = tok; break NAME; default: + warning(tok, + "Illegal #" + "pragma " + tok.getText()); return source_skipline(false); } } @@ -1360,7 +1385,7 @@ public class Preprocessor implements Closeable { final List<Token> value = new ArrayList<Token>(); VALUE: for (;;) { - tok = token(); + tok = source_token(); switch (tok.getType()) { case EOF: /* There ought to be a newline before EOF. @@ -1565,7 +1590,17 @@ public class Preprocessor implements Closeable { } } - private long expr(final int priority) + private int expr_char(Token token) { + Object value = token.getValue(); + if (value instanceof Character) + return ((Character) value).charValue(); + String text = String.valueOf(value); + if (text.length() == 0) + return 0; + return text.charAt(0); + } + + private long expr(int priority) throws IOException, LexerException { /* @@ -1601,7 +1636,7 @@ public class Preprocessor implements Closeable { lhs = value.longValue(); break; case CHARACTER: - lhs = ((Character) tok.getValue()).charValue(); + lhs = expr_char(tok); break; case IDENTIFIER: if (warnings.contains(Warning.UNDEF)) @@ -2128,6 +2163,8 @@ public class Preprocessor implements Closeable { * Returns the next preprocessor token. * * @see Token + * @return The next fully preprocessed token. + * @throws IOException if an I/O error occurs. * @throws LexerException if a preprocessing error occurs. * @throws InternalException if an unexpected error condition arises. */ diff --git a/src/main/java/com/jogamp/gluegen/jcpp/PreprocessorListener.java b/src/main/java/com/jogamp/gluegen/jcpp/PreprocessorListener.java index 1152a58..1feb1d6 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/PreprocessorListener.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/PreprocessorListener.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/ResourceFileSystem.java b/src/main/java/com/jogamp/gluegen/jcpp/ResourceFileSystem.java index 438d9c1..83cd248 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/ResourceFileSystem.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/ResourceFileSystem.java @@ -5,10 +5,9 @@ */ package com.jogamp.gluegen.jcpp; -import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; +import java.nio.charset.Charset; import javax.annotation.Nonnull; /** @@ -18,9 +17,11 @@ import javax.annotation.Nonnull; public class ResourceFileSystem implements VirtualFileSystem { private final ClassLoader loader; + private final Charset charset; - public ResourceFileSystem(@Nonnull ClassLoader loader) { + public ResourceFileSystem(@Nonnull ClassLoader loader, @Nonnull Charset charset) { this.loader = loader; + this.charset = charset; } @Override @@ -74,8 +75,7 @@ public class ResourceFileSystem implements VirtualFileSystem { @Override public Source getSource() throws IOException { InputStream stream = loader.getResourceAsStream(path); - BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); - return new LexerSource(reader, true); + return new InputLexerSource(stream, charset); } } } diff --git a/src/main/java/com/jogamp/gluegen/jcpp/Source.java b/src/main/java/com/jogamp/gluegen/jcpp/Source.java index d56b5d7..e254190 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/Source.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/Source.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/SourceIterator.java b/src/main/java/com/jogamp/gluegen/jcpp/SourceIterator.java index 4990512..db29822 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/SourceIterator.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/SourceIterator.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Iterator; import java.util.NoSuchElementException; +import javax.annotation.Nonnull; import static com.jogamp.gluegen.jcpp.Token.EOF; /** @@ -31,7 +32,7 @@ public class SourceIterator implements Iterator<Token> { private final Source source; private Token tok; - public SourceIterator(Source s) { + public SourceIterator(@Nonnull Source s) { this.source = s; this.tok = null; } @@ -57,6 +58,7 @@ public class SourceIterator implements Iterator<Token> { * @throws IllegalStateException if the Source * throws a LexerException or IOException */ + @Override public boolean hasNext() { advance(); return tok.getType() != EOF; @@ -69,6 +71,7 @@ public class SourceIterator implements Iterator<Token> { * @throws IllegalStateException if the Source * throws a LexerException or IOException */ + @Override public Token next() { if (!hasNext()) throw new NoSuchElementException(); @@ -80,8 +83,9 @@ public class SourceIterator implements Iterator<Token> { /** * Not supported. * - * @throws UnsupportedOperationException. + * @throws UnsupportedOperationException unconditionally. */ + @Override public void remove() { throw new UnsupportedOperationException(); } diff --git a/src/main/java/com/jogamp/gluegen/jcpp/State.java b/src/main/java/com/jogamp/gluegen/jcpp/State.java index 4d7f886..dc96006 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/State.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/State.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/StringLexerSource.java b/src/main/java/com/jogamp/gluegen/jcpp/StringLexerSource.java index 568bdca..a26fbe3 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/StringLexerSource.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/StringLexerSource.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +29,7 @@ public class StringLexerSource extends LexerSource { /** * Creates a new Source for lexing the given String. * + * @param string The input string to lex. * @param ppvalid true if preprocessor directives are to be * honoured within the string. */ @@ -39,8 +40,12 @@ public class StringLexerSource extends LexerSource { /** * Creates a new Source for lexing the given String. * + * Equivalent to calling <code>new StringLexerSource(string, false)</code>. + * * By default, preprocessor directives are not honoured within * the string. + * + * @param string The input string to lex. */ public StringLexerSource(String string) { this(string, false); diff --git a/src/main/java/com/jogamp/gluegen/jcpp/Token.java b/src/main/java/com/jogamp/gluegen/jcpp/Token.java index 9c1794b..0167bd6 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/Token.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/Token.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,9 @@ */ package com.jogamp.gluegen.jcpp; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; + /** * A Preprocessor token. * @@ -57,6 +60,9 @@ public final class Token { /** * Returns the semantic type of this token. + * + * @return the semantic type of this token. + * @see #getTokenName(int) */ public int getType() { return type; @@ -70,8 +76,12 @@ public final class Token { /** * Returns the line at which this token started. * - * Lines are numbered from zero. + * Lines are numbered from 1. + * + * @return the line at which this token started. + * @see LexerSource#getLine() */ + // Not @Nonnegative - might not have been assigned? public int getLine() { return line; } @@ -79,8 +89,12 @@ public final class Token { /** * Returns the column at which this token started. * - * Columns are numbered from zero. + * Columns are numbered from 0. + * + * @return the column at which this token started. + * @see LexerSource#getColumn() */ + // Not @Nonnegative - might not have been assigned? public int getColumn() { return column; } @@ -90,8 +104,10 @@ public final class Token { * * This is distinct from the semantic value of the token. * + * @return the original or generated text of this token. * @see #getValue() */ + // Not @Nonnull - might not have been assigned? public String getText() { return text; } @@ -103,8 +119,10 @@ public final class Token { * For integers, this is an Integer object. * For other token types, as appropriate. * + * @return the semantic value of this token, or null. * @see #getText() */ + // @CheckForNull // Not useful to annotate, as we have usually checked the type before calling this. public Object getValue() { return value; } @@ -138,7 +156,12 @@ public final class Token { * Returns the descriptive name of the given token type. * * This is mostly used for stringification and debugging. + * + * @param type The type constant from this class to name. + * @return the descriptive name of the given token type. + * @see Token#getType() */ + @Nonnull public static String getTokenName(int type) { return TokenType.getTokenName(type); } diff --git a/src/main/java/com/jogamp/gluegen/jcpp/TokenSnifferSource.java b/src/main/java/com/jogamp/gluegen/jcpp/TokenSnifferSource.java index cc60698..7c49072 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/TokenSnifferSource.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/TokenSnifferSource.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/VirtualFile.java b/src/main/java/com/jogamp/gluegen/jcpp/VirtualFile.java index 26bbbcb..a453890 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/VirtualFile.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/VirtualFile.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/VirtualFileSystem.java b/src/main/java/com/jogamp/gluegen/jcpp/VirtualFileSystem.java index 3f72f0c..a377753 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/VirtualFileSystem.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/VirtualFileSystem.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/jogamp/gluegen/jcpp/Warning.java b/src/main/java/com/jogamp/gluegen/jcpp/Warning.java index 89388f4..9e39e7e 100644 --- a/src/main/java/com/jogamp/gluegen/jcpp/Warning.java +++ b/src/main/java/com/jogamp/gluegen/jcpp/Warning.java @@ -1,6 +1,6 @@ /* * Anarres C Preprocessor - * Copyright (c) 2007-2008, Shevek + * Copyright (c) 2007-2015, Shevek * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/test/java/com/jogamp/gluegen/jcpp/LexerSourceTest.java b/src/test/java/com/jogamp/gluegen/jcpp/LexerSourceTest.java index 99324ff..4283073 100644 --- a/src/test/java/com/jogamp/gluegen/jcpp/LexerSourceTest.java +++ b/src/test/java/com/jogamp/gluegen/jcpp/LexerSourceTest.java @@ -144,4 +144,9 @@ public class LexerSourceTest extends SingletonJunitCase { testLexerSource("5 /*", false, NUMBER, WHITESPACE, INVALID); // Bug #15 testLexerSource("5 //", false, NUMBER, WHITESPACE, CPPCOMMENT); } + + @Test + public void testUnicode()throws Exception{ + testLexerSource("foo \u2018bar\u2019 baz", true, IDENTIFIER, WHITESPACE, 8216, IDENTIFIER, 8217, WHITESPACE, IDENTIFIER); + } } diff --git a/src/test/java/com/jogamp/gluegen/jcpp/NumericValueTest.java b/src/test/java/com/jogamp/gluegen/jcpp/NumericValueTest.java index 84a0ff1..036ee93 100644 --- a/src/test/java/com/jogamp/gluegen/jcpp/NumericValueTest.java +++ b/src/test/java/com/jogamp/gluegen/jcpp/NumericValueTest.java @@ -82,6 +82,7 @@ public class NumericValueTest extends SingletonJunitCase { testNumericValue("1e1", 1e1); // testNumericValue("-1e1", -1e1); testNumericValue("1e-1", 1e-1); + testNumericValue("1e+1", 1e+1); // Hex numbers with decimal exponents testNumericValue("0x12e3", 0x12e3); diff --git a/src/test/resources/lines.c b/src/test/resources/lines.c new file mode 100644 index 0000000..e0fc75c --- /dev/null +++ b/src/test/resources/lines.c @@ -0,0 +1,9 @@ +lines-c-line-1 +#include <lines1.h> +lines-c-line-3 + +lines-c-line-5 + +#include <lines2.h> + +lines-c-line-9 diff --git a/src/test/resources/lines1.h b/src/test/resources/lines1.h new file mode 100644 index 0000000..934a3a9 --- /dev/null +++ b/src/test/resources/lines1.h @@ -0,0 +1,18 @@ + +lines1-h-line-2 + +/* multi +line +comment +*/ + +#include <lines2.h> +lines1-h-line-10 + +lines1-h-line-12 +#include <lines2.h> + +/* trailing multiline comment +with +more lines +*/ diff --git a/src/test/resources/lines2.h b/src/test/resources/lines2.h new file mode 100644 index 0000000..0a71f2f --- /dev/null +++ b/src/test/resources/lines2.h @@ -0,0 +1,10 @@ + +lines2-h-line-2 + +/* +comment + +blank and multiline*/ + +lines2-h-line-9 + diff --git a/src/test/resources/pragma.c b/src/test/resources/pragma.c new file mode 100644 index 0000000..6018958 --- /dev/null +++ b/src/test/resources/pragma.c @@ -0,0 +1,8 @@ + +#pragma +#pragma once +#pragma foo(bar) +#pragma #pragma +#pragma #include <stdio.h> +#pragma #include + diff --git a/src/test/resources/regression/lex-char.in b/src/test/resources/regression/lex-char.in new file mode 100644 index 0000000..472af63 --- /dev/null +++ b/src/test/resources/regression/lex-char.in @@ -0,0 +1,4 @@ +#define EXAMPLE_X 'a' + +#if EXAMPLE == EXAMPLE_X +#endif |