Support separators in floating point literals

NOTICE

@@ -24,6 +24,7 @@ under the licensing terms detailed in LICENSE:
 * Julien Letellier <[email protected]>
 * Guido Zuidhof <[email protected]>
 * ncave <[email protected]>
+* Andrew Davis <[email protected]>
 
 Portions of this software are derived from third-party works licensed under
 the following terms:

src/tokenizer.ts

@@ -1462,18 +1462,13 @@ export class Tokenizer extends DiagnosticEmitter {
   }
 
   readDecimalFloat(): f64 {
-    // TODO: numeric separators (parseFloat can't handle these)
     var start = this.pos;
     var end = this.end;
     var text = this.source.text;
-    while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) {
-      ++this.pos;
-    }
+    this.readDecimalFloatPartial();
     if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) {
       ++this.pos;
-      while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) {
-        ++this.pos;
-      }
+      this.readDecimalFloatPartial();
     }
     if (this.pos < end) {
       let c = text.charCodeAt(this.pos);
@@ -1485,12 +1480,45 @@ export class Tokenizer extends DiagnosticEmitter {
         ) {
           ++this.pos;
         }
-        while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) {
-          ++this.pos;
+        this.readDecimalFloatPartial();
+      }
+    }
+    let floatString = text.substring(start, this.pos).replaceAll("_", "");
+    return parseFloat(floatString);
+  }
+
+  private readDecimalFloatPartial(): void {
+    let start = this.pos;
+    let end = this.end;
+    let text = this.source.text;
+    var sepEnd = start;
+
+    while (this.pos < end) {
+      let c = text.charCodeAt(this.pos);
+
+      if (c == CharCode._) {
+        if (sepEnd == this.pos) {
+          this.error(
+            sepEnd == start
+              ? DiagnosticCode.Numeric_separators_are_not_allowed_here
+              : DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted,
+            this.range(this.pos)
+          );
         }
+        sepEnd = this.pos + 1;
+      } else if (!isDecimalDigit(c)) {
+        break;
       }
+
+      ++this.pos;
+    }
+
+    if (this.pos != start && sepEnd == this.pos) {
+      this.error(
+        DiagnosticCode.Numeric_separators_are_not_allowed_here,
+        this.range(sepEnd - 1)
+      );
     }
-    return parseFloat(text.substring(start, this.pos));
   }
 
   readHexFloat(): f64 {

src/tsconfig.json

@@ -4,7 +4,7 @@
     "outDir": "../out",
     "allowJs": false,
     "sourceMap": true,
-    "target": "ES2016",
+    "target": "esnext",
     "strict": true
   },
   "include": [

tests/parser/numeric-separators.ts

@@ -2,6 +2,9 @@
 0b01_01_01;
 0o12_12_12;
 0x23_23_23;
+1_000_000.1234_1234;
+1_0e1_0;
+1_000_000e-1_0;
 
 // error cases that should still continue parsing:
 
@@ -16,3 +19,15 @@
 
 0x23_23_23_; // 6188
 0x23__23_23; // 6189
+
+1000_.1234; // 6188
+1000._1234; // 6188
+1000.1234_; // 6188
+
+10__00.1234; // 6189
+1000.12__34; // 6189
+
+1_e2; // 6188
+1e_2; // 6188
+1e2_; // 6188
+1e-1__0; // 6189

tests/parser/numeric-separators.ts.fixture.ts

@@ -2,6 +2,9 @@
 21;
 41610;
 2302755;
+1000000.12341234;
+100000000000;
+0.0001;
 111111;
 111111;
 21;
@@ -10,11 +13,29 @@
 41610;
 2302755;
 2302755;
-// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(8,9+0)
-// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(9,4+0)
-// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(11,11+0)
-// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(12,6+0)
+1000.1234;
+1000.1234;
+1000.1234;
+1000.1234;
+1000.1234;
+100;
+100;
+100;
+1e-10;
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(11,9+0)
+// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(12,4+0)
 // ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(14,11+0)
 // ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(15,6+0)
 // ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(17,11+0)
 // ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(18,6+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(20,11+0)
+// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(21,6+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(23,5+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(24,6+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(25,10+0)
+// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(27,4+0)
+// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(28,9+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(30,2+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(31,3+0)
+// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(32,4+0)
+// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(33,6+0)