Emit all strings with extended escapes using the same scheme as templates.

DanielRosenwasser · DanielRosenwasser · commit 4657c2dfd505 · 2015-02-27T12:55:46.000-08:00
diff --git a/src/compiler/emitter.ts b/src/compiler/emitter.ts
@@ -2187,9 +2187,12 @@ module ts {
             }
 
             function emitLiteral(node: LiteralExpression) {
-                var text = languageVersion < ScriptTarget.ES6 && isTemplateLiteralKind(node.kind) ? getTemplateLiteralAsStringLiteral(node) :
-                    node.parent ? getSourceTextOfNodeFromSourceFile(currentSourceFile, node) :
-                        node.text;
+                var text = languageVersion < ScriptTarget.ES6 && (isTemplateLiteralKind(node.kind) || node.hasExtendedUnicodeEscape)
+                    ? getDoubleQuotedStringTextOfLiteral(node)
+                    : node.parent
+                        ? getSourceTextOfNodeFromSourceFile(currentSourceFile, node)
+                        : node.text; // TODO(drosen): Is this correct?
+                
                 if (compilerOptions.sourceMap && (node.kind === SyntaxKind.StringLiteral || isTemplateLiteralKind(node.kind))) {
                     writer.writeLiteral(text);
                 }
@@ -2202,7 +2205,7 @@ module ts {
                 }
             }
             
-            function getTemplateLiteralAsStringLiteral(node: LiteralExpression): string {
+            function getDoubleQuotedStringTextOfLiteral(node: LiteralExpression): string {
                 var result = escapeString(node.text);
                 result = replaceNonAsciiCharacters(result);
                 
diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts
@@ -2164,6 +2164,10 @@ module ts {
             var text = scanner.getTokenValue();
             node.text = internName ? internIdentifier(text) : text;
 
+            if (scanner.hasExtendedUnicodeEscape()) {
+                node.hasExtendedUnicodeEscape = true;
+            }
+            
             if (scanner.isUnterminated()) {
                 node.isUnterminated = true;
             }
diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
@@ -14,6 +14,7 @@ module ts {
         getTokenPos(): number;
         getTokenText(): string;
         getTokenValue(): string;
+        hasExtendedUnicodeEscape(): boolean;
         hasPrecedingLineBreak(): boolean;
         isIdentifier(): boolean;
         isReservedWord(): boolean;
@@ -556,6 +557,7 @@ module ts {
         var token: SyntaxKind;
         var tokenValue: string;
         var precedingLineBreak: boolean;
+        var hasExtendedUnicodeEscape: boolean;
         var tokenIsUnterminated: boolean;
 
         function error(message: DiagnosticMessage, length?: number): void {
@@ -776,6 +778,7 @@ module ts {
                     return "\"";
                 case CharacterCodes.u:
                     if (text.charCodeAt(pos) === CharacterCodes.openBrace) {
+                        hasExtendedUnicodeEscape = true;
                         pos++;
                         var escapedValue = scanMinimumNumberOfHexDigits(1);
                         
@@ -926,6 +929,7 @@ module ts {
 
         function scan(): SyntaxKind {
             startPos = pos;
+            hasExtendedUnicodeEscape = false;
             precedingLineBreak = false;
             tokenIsUnterminated = false;
             while (true) {
@@ -1393,6 +1397,7 @@ module ts {
             getTokenPos: () => tokenPos,
             getTokenText: () => text.substring(tokenPos, pos),
             getTokenValue: () => tokenValue,
+            hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape,
             hasPrecedingLineBreak: () => precedingLineBreak,
             isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord,
             isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord,
diff --git a/src/compiler/types.ts b/src/compiler/types.ts
@@ -655,6 +655,7 @@ module ts {
     export interface LiteralExpression extends PrimaryExpression {
         text: string;
         isUnterminated?: boolean;
+        hasExtendedUnicodeEscape?: boolean;
     }
 
     export interface StringLiteralExpression extends LiteralExpression {
diff --git a/tests/baselines/reference/APISample_compile.js b/tests/baselines/reference/APISample_compile.js
@@ -552,6 +552,7 @@ declare module "typescript" {
     interface LiteralExpression extends PrimaryExpression {
         text: string;
         isUnterminated?: boolean;
+        hasExtendedUnicodeEscape?: boolean;
     }
     interface StringLiteralExpression extends LiteralExpression {
         _stringLiteralExpressionBrand: any;
@@ -1421,6 +1422,7 @@ declare module "typescript" {
         getTokenPos(): number;
         getTokenText(): string;
         getTokenValue(): string;
+        hasExtendedUnicodeEscape(): boolean;
         hasPrecedingLineBreak(): boolean;
         isIdentifier(): boolean;
         isReservedWord(): boolean;
diff --git a/tests/baselines/reference/APISample_compile.types b/tests/baselines/reference/APISample_compile.types
@@ -1664,6 +1664,9 @@ declare module "typescript" {
 
         isUnterminated?: boolean;
 >isUnterminated : boolean
+
+        hasExtendedUnicodeEscape?: boolean;
+>hasExtendedUnicodeEscape : boolean
     }
     interface StringLiteralExpression extends LiteralExpression {
 >StringLiteralExpression : StringLiteralExpression
@@ -4481,6 +4484,9 @@ declare module "typescript" {
         getTokenValue(): string;
 >getTokenValue : () => string
 
+        hasExtendedUnicodeEscape(): boolean;
+>hasExtendedUnicodeEscape : () => boolean
+
         hasPrecedingLineBreak(): boolean;
 >hasPrecedingLineBreak : () => boolean
 
diff --git a/tests/baselines/reference/APISample_linter.js b/tests/baselines/reference/APISample_linter.js
@@ -583,6 +583,7 @@ declare module "typescript" {
     interface LiteralExpression extends PrimaryExpression {
         text: string;
         isUnterminated?: boolean;
+        hasExtendedUnicodeEscape?: boolean;
     }
     interface StringLiteralExpression extends LiteralExpression {
         _stringLiteralExpressionBrand: any;
@@ -1452,6 +1453,7 @@ declare module "typescript" {
         getTokenPos(): number;
         getTokenText(): string;
         getTokenValue(): string;
+        hasExtendedUnicodeEscape(): boolean;
         hasPrecedingLineBreak(): boolean;
         isIdentifier(): boolean;
         isReservedWord(): boolean;
diff --git a/tests/baselines/reference/APISample_linter.types b/tests/baselines/reference/APISample_linter.types
@@ -1810,6 +1810,9 @@ declare module "typescript" {
 
         isUnterminated?: boolean;
 >isUnterminated : boolean
+
+        hasExtendedUnicodeEscape?: boolean;
+>hasExtendedUnicodeEscape : boolean
     }
     interface StringLiteralExpression extends LiteralExpression {
 >StringLiteralExpression : StringLiteralExpression
@@ -4627,6 +4630,9 @@ declare module "typescript" {
         getTokenValue(): string;
 >getTokenValue : () => string
 
+        hasExtendedUnicodeEscape(): boolean;
+>hasExtendedUnicodeEscape : () => boolean
+
         hasPrecedingLineBreak(): boolean;
 >hasPrecedingLineBreak : () => boolean
 
diff --git a/tests/baselines/reference/APISample_transform.js b/tests/baselines/reference/APISample_transform.js
@@ -584,6 +584,7 @@ declare module "typescript" {
     interface LiteralExpression extends PrimaryExpression {
         text: string;
         isUnterminated?: boolean;
+        hasExtendedUnicodeEscape?: boolean;
     }
     interface StringLiteralExpression extends LiteralExpression {
         _stringLiteralExpressionBrand: any;
@@ -1453,6 +1454,7 @@ declare module "typescript" {
         getTokenPos(): number;
         getTokenText(): string;
         getTokenValue(): string;
+        hasExtendedUnicodeEscape(): boolean;
         hasPrecedingLineBreak(): boolean;
         isIdentifier(): boolean;
         isReservedWord(): boolean;
diff --git a/tests/baselines/reference/APISample_transform.types b/tests/baselines/reference/APISample_transform.types
@@ -1760,6 +1760,9 @@ declare module "typescript" {
 
         isUnterminated?: boolean;
 >isUnterminated : boolean
+
+        hasExtendedUnicodeEscape?: boolean;
+>hasExtendedUnicodeEscape : boolean
     }
     interface StringLiteralExpression extends LiteralExpression {
 >StringLiteralExpression : StringLiteralExpression
@@ -4577,6 +4580,9 @@ declare module "typescript" {
         getTokenValue(): string;
 >getTokenValue : () => string
 
+        hasExtendedUnicodeEscape(): boolean;
+>hasExtendedUnicodeEscape : () => boolean
+
         hasPrecedingLineBreak(): boolean;
 >hasPrecedingLineBreak : () => boolean
 
diff --git a/tests/baselines/reference/APISample_watcher.js b/tests/baselines/reference/APISample_watcher.js
@@ -621,6 +621,7 @@ declare module "typescript" {
     interface LiteralExpression extends PrimaryExpression {
         text: string;
         isUnterminated?: boolean;
+        hasExtendedUnicodeEscape?: boolean;
     }
     interface StringLiteralExpression extends LiteralExpression {
         _stringLiteralExpressionBrand: any;
@@ -1490,6 +1491,7 @@ declare module "typescript" {
         getTokenPos(): number;
         getTokenText(): string;
         getTokenValue(): string;
+        hasExtendedUnicodeEscape(): boolean;
         hasPrecedingLineBreak(): boolean;
         isIdentifier(): boolean;
         isReservedWord(): boolean;
diff --git a/tests/baselines/reference/APISample_watcher.types b/tests/baselines/reference/APISample_watcher.types
@@ -1933,6 +1933,9 @@ declare module "typescript" {
 
         isUnterminated?: boolean;
 >isUnterminated : boolean
+
+        hasExtendedUnicodeEscape?: boolean;
+>hasExtendedUnicodeEscape : boolean
     }
     interface StringLiteralExpression extends LiteralExpression {
 >StringLiteralExpression : StringLiteralExpression
@@ -4750,6 +4753,9 @@ declare module "typescript" {
         getTokenValue(): string;
 >getTokenValue : () => string
 
+        hasExtendedUnicodeEscape(): boolean;
+>hasExtendedUnicodeEscape : () => boolean
+
         hasPrecedingLineBreak(): boolean;
 >hasPrecedingLineBreak : () => boolean
 
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings01_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings01_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{0}";
 
 
 //// [unicodeExtendedEscapesInStrings01_ES5.js]
-var x = "\u{0}";
+var x = "\0";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings02_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings02_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{00}";
 
 
 //// [unicodeExtendedEscapesInStrings02_ES5.js]
-var x = "\u{00}";
+var x = "\0";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings03_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings03_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{0000}";
 
 
 //// [unicodeExtendedEscapesInStrings03_ES5.js]
-var x = "\u{0000}";
+var x = "\0";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings04_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings04_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{00000000}";
 
 
 //// [unicodeExtendedEscapesInStrings04_ES5.js]
-var x = "\u{00000000}";
+var x = "\0";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings05_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings05_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{48}\u{65}\u{6c}\u{6c}\u{6f}\u{20}\u{77}\u{6f}\u{72}\u{6c}\u{64}";
 
 
 //// [unicodeExtendedEscapesInStrings05_ES5.js]
-var x = "\u{48}\u{65}\u{6c}\u{6c}\u{6f}\u{20}\u{77}\u{6f}\u{72}\u{6c}\u{64}";
+var x = "Hello world";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings06_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings06_ES5.js
@@ -8,4 +8,4 @@ var x = "\u{10FFFF}";
 //// [unicodeExtendedEscapesInStrings06_ES5.js]
 // ES6 Spec - 10.1.1 Static Semantics: UTF16Encoding (cp)
 //  1. Assert: 0 ≤ cp ≤ 0x10FFFF.
-var x = "\u{10FFFF}";
+var x = "\uDBFF\uDFFF";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings07_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings07_ES5.js
@@ -8,4 +8,4 @@ var x = "\u{110000}";
 //// [unicodeExtendedEscapesInStrings07_ES5.js]
 // ES6 Spec - 10.1.1 Static Semantics: UTF16Encoding (cp)
 //  1. Assert: 0 ≤ cp ≤ 0x10FFFF.
-var x = "\u{110000}";
+var x = "}";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings08_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings08_ES5.js
@@ -10,4 +10,4 @@ var x = "\u{FFFF}";
 // ES6 Spec - 10.1.1 Static Semantics: UTF16Encoding (cp)
 //  2. If cp ≤ 65535, return cp.
 // (FFFF == 65535)
-var x = "\u{FFFF}";
+var x = "\uFFFF";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings09_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings09_ES5.js
@@ -10,4 +10,4 @@ var x = "\u{10000}";
 // ES6 Spec - 10.1.1 Static Semantics: UTF16Encoding (cp)
 //  2. If cp ≤ 65535, return cp.
 // (10000 == 65536)
-var x = "\u{10000}";
+var x = "\uD800\uDC00";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings10_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings10_ES5.js
@@ -12,4 +12,4 @@ var x = "\u{D800}";
 //  2. Let cu1 be floor((cp – 65536) / 1024) + 0xD800.
 // Although we should just get back a single code point value of 0xD800,
 // this is a useful edge-case test.
-var x = "\u{D800}";
+var x = "\uD800";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings11_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings11_ES5.js
@@ -12,4 +12,4 @@ var x = "\u{DC00}";
 //  2. Let cu2 be ((cp – 65536) modulo 1024) + 0xDC00.
 // Although we should just get back a single code point value of 0xDC00,
 // this is a useful edge-case test.
-var x = "\u{DC00}";
+var x = "\uDC00";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings12_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings12_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{FFFFFFFF}";
 
 
 //// [unicodeExtendedEscapesInStrings12_ES5.js]
-var x = "\u{FFFFFFFF}";
+var x = "}";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings13_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings13_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{DDDDD}";
 
 
 //// [unicodeExtendedEscapesInStrings13_ES5.js]
-var x = "\u{DDDDD}";
+var x = "\uDB37\uDDDD";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings14_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings14_ES5.js
@@ -6,4 +6,4 @@ var x = "\u{-DDDD}";
 
 //// [unicodeExtendedEscapesInStrings14_ES5.js]
 // Shouldn't work, negatives are not allowed.
-var x = "\u{-DDDD}";
+var x = "-DDDD}";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings15_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings15_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{abcd}\u{ef12}\u{3456}\u{7890}";
 
 
 //// [unicodeExtendedEscapesInStrings15_ES5.js]
-var x = "\u{abcd}\u{ef12}\u{3456}\u{7890}";
+var x = "\uABCD\uEF12\u3456\u7890";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings16_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings16_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{ABCD}\u{EF12}\u{3456}\u{7890}";
 
 
 //// [unicodeExtendedEscapesInStrings16_ES5.js]
-var x = "\u{ABCD}\u{EF12}\u{3456}\u{7890}";
+var x = "\uABCD\uEF12\u3456\u7890";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings17_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings17_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{r}\u{n}\u{t}";
 
 
 //// [unicodeExtendedEscapesInStrings17_ES5.js]
-var x = "\u{r}\u{n}\u{t}";
+var x = "r}n}t}";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings18_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings18_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{65}\u{65}";
 
 
 //// [unicodeExtendedEscapesInStrings18_ES5.js]
-var x = "\u{65}\u{65}";
+var x = "ee";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings19_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings19_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{}";
 
 
 //// [unicodeExtendedEscapesInStrings19_ES5.js]
-var x = "\u{}";
+var x = "}";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings20_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings20_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{";
 
 
 //// [unicodeExtendedEscapesInStrings20_ES5.js]
-var x = "\u{";
+var x = "";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings21_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings21_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{67";
 
 
 //// [unicodeExtendedEscapesInStrings21_ES5.js]
-var x = "\u{67";
+var x = "g";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings22_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings22_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{00000000000067";
 
 
 //// [unicodeExtendedEscapesInStrings22_ES5.js]
-var x = "\u{00000000000067";
+var x = "g";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings23_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings23_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{00000000000067}";
 
 
 //// [unicodeExtendedEscapesInStrings23_ES5.js]
-var x = "\u{00000000000067}";
+var x = "g";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings24_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings24_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{00000000000067
 
 
 //// [unicodeExtendedEscapesInStrings24_ES5.js]
-var x = "\u{00000000000067;
+var x = "g";
diff --git a/tests/baselines/reference/unicodeExtendedEscapesInStrings25_ES5.js b/tests/baselines/reference/unicodeExtendedEscapesInStrings25_ES5.js
@@ -4,4 +4,4 @@ var x = "\u{00000000000067}
 
 
 //// [unicodeExtendedEscapesInStrings25_ES5.js]
-var x = "\u{00000000000067};
+var x = "g";

Original file line number	Diff line number	Diff line change
`@@ -2187,9 +2187,12 @@ module ts {`
`2187`	`2187`	`}`
`2188`	`2188`
`2189`	`2189`	`function emitLiteral(node: LiteralExpression) {`
`2190`		`- var text = languageVersion < ScriptTarget.ES6 && isTemplateLiteralKind(node.kind) ? getTemplateLiteralAsStringLiteral(node) :`
`2191`		`- node.parent ? getSourceTextOfNodeFromSourceFile(currentSourceFile, node) :`
`2192`		`- node.text;`
	`2190`	`+ var text = languageVersion < ScriptTarget.ES6 && (isTemplateLiteralKind(node.kind) \|\| node.hasExtendedUnicodeEscape)`
	`2191`	`+ ? getDoubleQuotedStringTextOfLiteral(node)`
	`2192`	`+ : node.parent`
	`2193`	`+ ? getSourceTextOfNodeFromSourceFile(currentSourceFile, node)`
	`2194`	`+ : node.text; // TODO(drosen): Is this correct?`
	`2195`	`+`
`2193`	`2196`	`if (compilerOptions.sourceMap && (node.kind === SyntaxKind.StringLiteral \|\| isTemplateLiteralKind(node.kind))) {`
`2194`	`2197`	`writer.writeLiteral(text);`
`2195`	`2198`	`}`
`@@ -2202,7 +2205,7 @@ module ts {`
`2202`	`2205`	`}`
`2203`	`2206`	`}`
`2204`	`2207`
`2205`		`- function getTemplateLiteralAsStringLiteral(node: LiteralExpression): string {`
	`2208`	`+ function getDoubleQuotedStringTextOfLiteral(node: LiteralExpression): string {`
`2206`	`2209`	`var result = escapeString(node.text);`
`2207`	`2210`	`result = replaceNonAsciiCharacters(result);`
`2208`	`2211`
Original file line number	Diff line number	Diff line change
`@@ -655,6 +655,7 @@ module ts {`
`655`	`655`	`export interface LiteralExpression extends PrimaryExpression {`
`656`	`656`	`text: string;`
`657`	`657`	`isUnterminated?: boolean;`
	`658`	`+ hasExtendedUnicodeEscape?: boolean;`
`658`	`659`	`}`
`659`	`660`
`660`	`661`	`export interface StringLiteralExpression extends LiteralExpression {`