Fix floating point lexing: Missing chars and bad token stream.

2 files changed, 41 insertions, 19 deletions

diff --git a/src/main/java/org/anarres/cpp/LexerSource.java b/src/main/java/org/anarres/cpp/LexerSource.java
index ca18314..5f1dac3 100644
--- a/src/main/java/org/anarres/cpp/LexerSource.java
+++ b/src/main/java/org/anarres/cpp/LexerSource.java
@@ -587,11 +587,13 @@ public class LexerSource extends Source {
         NumericValue value = new NumericValue(10, integer);
         int d = read();
         if (d == '.') {
+            text.append((char) d);
             String fraction = _number_part(text, 10);
             value.setFractionalPart(fraction);
             d = read();
         }
         if (d == 'E' || d == 'e') {
+            text.append((char) d);
             String exponent = _number_part(text, 10);
             value.setExponent(exponent);
             d = read();
@@ -847,7 +849,11 @@ public class LexerSource extends Source {
                 d = read();
                 if (d == 'x' || d == 'X')
                     tok = number_hex((char) d);
-                else {
+                else if (d == '.') {
+                    unread(d);
+                    unread(c);
+                    tok = number_decimal();
+                } else {
                     unread(d);
                     tok = number_octal();
                 }
diff --git a/src/test/java/org/anarres/cpp/LexerSourceTest.java b/src/test/java/org/anarres/cpp/LexerSourceTest.java
index 76bc673..f51b253 100644
--- a/src/test/java/org/anarres/cpp/LexerSourceTest.java
+++ b/src/test/java/org/anarres/cpp/LexerSourceTest.java
@@ -7,47 +7,52 @@ import static org.junit.Assert.*;
 
 public class LexerSourceTest {
 
-    private void testLexerSource(String in, int... out)
+    private void testLexerSource(String in, boolean textmatch, int... out)
             throws Exception {
         System.out.println("Testing '" + in + "' => "
                 + Arrays.toString(out));
         StringLexerSource s = new StringLexerSource(in);
 
-        int col = 0;
+        StringBuilder buf = new StringBuilder();
         for (int i = 0; i < out.length; i++) {
             Token tok = s.token();
             System.out.println("Token is " + tok);
             assertEquals(out[i], tok.getType());
             // assertEquals(col, tok.getColumn());
-            col += tok.getText().length();
+            buf.append(tok.getText());
         }
 
         Token tok = s.token();
         System.out.println("Token is " + tok);
         assertEquals(EOF, tok.getType());
+
+        if (textmatch)
+            assertEquals(in, buf.toString());
     }
 
     @Test
     public void testLexerSource()
             throws Exception {
 
-        testLexerSource("int a = 5;",
+        testLexerSource("int a = 5;", true,
                 IDENTIFIER, WHITESPACE, IDENTIFIER, WHITESPACE,
-                '=', WHITESPACE, NUMBER, ';', EOF
+                '=', WHITESPACE, NUMBER, ';'
         );
 
         // \n is WHITESPACE because ppvalid = false
-        testLexerSource("# #   \r\n\n\r \rfoo",
+        testLexerSource("# #   \r\n\n\r \rfoo", true,
                 HASH, WHITESPACE, '#', WHITESPACE, IDENTIFIER
         );
 
-        testLexerSource("%:%:", PASTE);
-        testLexerSource("%:?", '#', '?');
-        testLexerSource("%:%=", '#', MOD_EQ);
-        testLexerSource("0x1234ffdUL 0765I",
+        // No match - trigraphs
+        testLexerSource("%:%:", false, PASTE);
+        testLexerSource("%:?", false, '#', '?');
+        testLexerSource("%:%=", false, '#', MOD_EQ);
+
+        testLexerSource("0x1234ffdUL 0765I", true,
                 NUMBER, WHITESPACE, NUMBER);
 
-        testLexerSource("+= -= *= /= %= <= >= >>= <<= &= |= ^= x",
+        testLexerSource("+= -= *= /= %= <= >= >>= <<= &= |= ^= x", true,
                 PLUS_EQ, WHITESPACE,
                 SUB_EQ, WHITESPACE,
                 MULT_EQ, WHITESPACE,
@@ -62,11 +67,11 @@ public class LexerSourceTest {
                 XOR_EQ, WHITESPACE,
                 IDENTIFIER);
 
-        testLexerSource("/**/", CCOMMENT);
-        testLexerSource("/* /**/ */", CCOMMENT, WHITESPACE, '*', '/');
-        testLexerSource("/** ** **/", CCOMMENT);
-        testLexerSource("//* ** **/", CPPCOMMENT);
-        testLexerSource("'\\r' '\\xf' '\\xff' 'x' 'aa' ''",
+        testLexerSource("/**/", true, CCOMMENT);
+        testLexerSource("/* /**/ */", true, CCOMMENT, WHITESPACE, '*', '/');
+        testLexerSource("/** ** **/", true, CCOMMENT);
+        testLexerSource("//* ** **/", true, CPPCOMMENT);
+        testLexerSource("'\\r' '\\xf' '\\xff' 'x' 'aa' ''", true,
                 CHARACTER, WHITESPACE,
                 CHARACTER, WHITESPACE,
                 CHARACTER, WHITESPACE,
@@ -74,13 +79,24 @@ public class LexerSourceTest {
                 SQSTRING, WHITESPACE,
                 SQSTRING);
 
-        testLexerSource("1i1I1l1L1ui1ul",
+        testLexerSource("1i1I1l1L1ui1ul", true,
                 NUMBER, NUMBER,
                 NUMBER, NUMBER,
                 NUMBER, NUMBER);
 
-        testLexerSource("'' 'x' 'xx'",
+        testLexerSource("'' 'x' 'xx'", true,
                 SQSTRING, WHITESPACE, CHARACTER, WHITESPACE, SQSTRING);
     }
 
+    @Test
+    public void testNumbers() throws Exception {
+        testLexerSource("0", true, NUMBER);
+        testLexerSource("045", true, NUMBER);
+        testLexerSource("45", true, NUMBER);
+        testLexerSource("0.45", true, NUMBER);
+        testLexerSource("1.45", true, NUMBER);
+        testLexerSource("1e6", true, NUMBER);
+        testLexerSource("1.45e6", true, NUMBER);
+        testLexerSource(".45e6", true, NUMBER);
+    }
 }