diff --git a/autoload/vimlparser.vim b/autoload/vimlparser.vim index f3955c53..575f9dcf 100644 --- a/autoload/vimlparser.vim +++ b/autoload/vimlparser.vim @@ -140,6 +140,7 @@ let s:NODE_LAMBDA = 92 let s:NODE_BLOB = 93 let s:NODE_CONST = 94 let s:NODE_EVAL = 95 +let s:NODE_HEREDOC = 96 let s:TOKEN_EOF = 1 let s:TOKEN_EOL = 2 @@ -209,6 +210,7 @@ let s:TOKEN_ARROW = 65 let s:TOKEN_BLOB = 66 let s:TOKEN_LITCOPEN = 67 let s:TOKEN_DOTDOT = 68 +let s:TOKEN_HEREDOC = 69 let s:MAX_FUNC_ARGS = 20 @@ -412,6 +414,7 @@ endfunction " CURLYNAMEPART .value " CURLYNAMEEXPR .value " LAMBDA .rlist .left +" HEREDOC .rlist .op .body function! s:Node(type) abort return {'type': a:type} endfunction @@ -1500,6 +1503,44 @@ function! s:VimLParser.parse_cmd_call() abort call self.add_node(node) endfunction +function! s:VimLParser.parse_heredoc() abort + let node = s:Node(s:NODE_HEREDOC) + let node.pos = self.ea.cmdpos + let node.op = '' + let node.rlist = [] + let node.body = [] + + while s:TRUE + call self.reader.skip_white() + let key = self.reader.read_word() + if key == '' + break + endif + if !s:islower(key[0]) + let node.op = key + break + else + call add(node.rlist, key) + endif + endwhile + if node.op ==# '' + throw s:Err('E172: Missing marker', self.reader.getpos()) + endif + call self.parse_trail() + while s:TRUE + if self.reader.peek() ==# '' + break + endif + let line = self.reader.getn(-1) + if line ==# node.op + return node + endif + call add(node.body, line) + call self.reader.get() + endwhile + throw s:Err(printf("E990: Missing end marker '%s'", node.op), self.reader.getpos()) +endfunction + function! s:VimLParser.parse_cmd_let() abort let pos = self.reader.tell() call self.reader.skip_white() @@ -1518,10 +1559,12 @@ function! s:VimLParser.parse_cmd_let() abort " TODO check scriptversion? if s2 ==# '..' let s2 = self.reader.peekn(3) + elseif s2 ==# '=<' + let s2 = self.reader.peekn(3) endif " :let {var-name} .. - if self.ends_excmds(s1) || (s2 !=# '+=' && s2 !=# '-=' && s2 !=# '.=' && s2 !=# '..=' && s2 !=# '*=' && s2 !=# '/=' && s2 !=# '%=' && s1 !=# '=') + if self.ends_excmds(s1) || (s2 !=# '+=' && s2 !=# '-=' && s2 !=# '.=' && s2 !=# '..=' && s2 !=# '*=' && s2 !=# '/=' && s2 !=# '%=' && s2 !=# '=<<' && s1 !=# '=') call self.reader.seek_set(pos) call self.parse_cmd_common() return @@ -1539,6 +1582,13 @@ function! s:VimLParser.parse_cmd_let() abort if s2 ==# '+=' || s2 ==# '-=' || s2 ==# '.=' || s2 ==# '..=' || s2 ==# '*=' || s2 ==# '/=' || s2 ==# '%=' call self.reader.getn(len(s2)) let node.op = s2 + elseif s2 ==# '=<<' + call self.reader.getn(len(s2)) + call self.reader.skip_white() + let node.op = s2 + let node.right = self.parse_heredoc() + call self.add_node(node) + return elseif s1 ==# '=' call self.reader.getn(1) let node.op = s1 @@ -4958,6 +5008,8 @@ function! s:Compiler.compile(node) abort return self.compile_curlynameexpr(a:node) elseif a:node.type ==# s:NODE_LAMBDA return self.compile_lambda(a:node) + elseif a:node.type == s:NODE_HEREDOC + return self.compile_heredoc(a:node) else throw printf('Compiler: unknown node: %s', string(a:node)) endif @@ -5451,11 +5503,41 @@ function! s:Compiler.compile_curlynameexpr(node) abort return '{' . self.compile(a:node.value) . '}' endfunction +function! s:Compiler.escape_string(str) abort + let m = {"\n": '\n', "\t": '\t', "\r": '\r'} + let out = '"' + for i in range(len(a:str)) + let c = a:str[i] + if has_key(m, c) + let out .= m[c] + else + let out .= c + endif + endfor + let out .= '"' + return out +endfunction + function! s:Compiler.compile_lambda(node) abort let rlist = map(a:node.rlist, 'self.compile(v:val)') return printf('(lambda (%s) %s)', join(rlist, ' '), self.compile(a:node.left)) endfunction +function! s:Compiler.compile_heredoc(node) abort + if empty(a:node.rlist) + let rlist = '(list)' + else + let rlist = '(list ' . join(map(a:node.rlist, 'self.escape_string(v:val)'), ' ') . ')' + endif + if empty(a:node.body) + let body = '(list)' + else + let body = '(list ' . join(map(a:node.body, 'self.escape_string(v:val)'), ' ') . ')' + endif + let op = self.escape_string(a:node.op) + return printf('(heredoc %s %s %s)', rlist, op, body) +endfunction + " TODO: under construction let s:RegexpParser = {} diff --git a/js/vimlparser.js b/js/vimlparser.js index 94fe7ede..9a017022 100644 --- a/js/vimlparser.js +++ b/js/vimlparser.js @@ -321,6 +321,7 @@ var NODE_LAMBDA = 92; var NODE_BLOB = 93; var NODE_CONST = 94; var NODE_EVAL = 95; +var NODE_HEREDOC = 96; var TOKEN_EOF = 1; var TOKEN_EOL = 2; var TOKEN_SPACE = 3; @@ -389,6 +390,7 @@ var TOKEN_ARROW = 65; var TOKEN_BLOB = 66; var TOKEN_LITCOPEN = 67; var TOKEN_DOTDOT = 68; +var TOKEN_HEREDOC = 69; var MAX_FUNC_ARGS = 20; function isalpha(c) { return viml_eqregh(c, "^[A-Za-z]$"); @@ -590,6 +592,7 @@ function ExArg() { // CURLYNAMEPART .value // CURLYNAMEEXPR .value // LAMBDA .rlist .left +// HEREDOC .rlist .op .body function Node(type) { return {"type":type}; } @@ -1795,6 +1798,44 @@ VimLParser.prototype.parse_cmd_call = function() { this.add_node(node); } +VimLParser.prototype.parse_heredoc = function() { + var node = Node(NODE_HEREDOC); + node.pos = this.ea.cmdpos; + node.op = ""; + node.rlist = []; + node.body = []; + while (TRUE) { + this.reader.skip_white(); + var key = this.reader.read_word(); + if (key == "") { + break; + } + if (!islower(key[0])) { + node.op = key; + break; + } + else { + viml_add(node.rlist, key); + } + } + if (node.op == "") { + throw Err("E172: Missing marker", this.reader.getpos()); + } + this.parse_trail(); + while (TRUE) { + if (this.reader.peek() == "") { + break; + } + var line = this.reader.getn(-1); + if (line == node.op) { + return node; + } + viml_add(node.body, line); + this.reader.get(); + } + throw Err(viml_printf("E990: Missing end marker '%s'", node.op), this.reader.getpos()); +} + VimLParser.prototype.parse_cmd_let = function() { var pos = this.reader.tell(); this.reader.skip_white(); @@ -1812,8 +1853,11 @@ VimLParser.prototype.parse_cmd_let = function() { if (s2 == "..") { var s2 = this.reader.peekn(3); } + else if (s2 == "=<") { + var s2 = this.reader.peekn(3); + } // :let {var-name} .. - if (this.ends_excmds(s1) || s2 != "+=" && s2 != "-=" && s2 != ".=" && s2 != "..=" && s2 != "*=" && s2 != "/=" && s2 != "%=" && s1 != "=") { + if (this.ends_excmds(s1) || s2 != "+=" && s2 != "-=" && s2 != ".=" && s2 != "..=" && s2 != "*=" && s2 != "/=" && s2 != "%=" && s2 != "=<<" && s1 != "=") { this.reader.seek_set(pos); this.parse_cmd_common(); return; @@ -1831,6 +1875,14 @@ VimLParser.prototype.parse_cmd_let = function() { this.reader.getn(viml_len(s2)); node.op = s2; } + else if (s2 == "=<<") { + this.reader.getn(viml_len(s2)); + this.reader.skip_white(); + node.op = s2; + node.right = this.parse_heredoc(); + this.add_node(node); + return; + } else if (s1 == "=") { this.reader.getn(1); node.op = s1; @@ -4461,6 +4513,9 @@ Compiler.prototype.compile = function(node) { else if (node.type == NODE_LAMBDA) { return this.compile_lambda(node); } + else if (node.type == NODE_HEREDOC) { + return this.compile_heredoc(node); + } else { throw viml_printf("Compiler: unknown node: %s", viml_string(node)); } @@ -4973,11 +5028,46 @@ Compiler.prototype.compile_curlynameexpr = function(node) { return "{" + this.compile(node.value) + "}"; } +Compiler.prototype.escape_string = function(str) { + var m = {"\n":"\\n", "\t":"\\t", "\r":"\\r"}; + var out = "\""; + var __c14 = viml_range(viml_len(str)); + for (var __i14 = 0; __i14 < __c14.length; ++__i14) { + var i = __c14[__i14]; + var c = str[i]; + if (viml_has_key(m, c)) { + out += m[c]; + } + else { + out += c; + } + } + out += "\""; + return out; +} + Compiler.prototype.compile_lambda = function(node) { var rlist = node.rlist.map((function(vval) { return this.compile(vval); }).bind(this)); return viml_printf("(lambda (%s) %s)", viml_join(rlist, " "), this.compile(node.left)); } +Compiler.prototype.compile_heredoc = function(node) { + if (viml_empty(node.rlist)) { + var rlist = "(list)"; + } + else { + var rlist = "(list " + viml_join(node.rlist.map((function(vval) { return this.escape_string(vval); }).bind(this)), " ") + ")"; + } + if (viml_empty(node.body)) { + var body = "(list)"; + } + else { + var body = "(list " + viml_join(node.body.map((function(vval) { return this.escape_string(vval); }).bind(this)), " ") + ")"; + } + var op = this.escape_string(node.op); + return viml_printf("(heredoc %s %s %s)", rlist, op, body); +} + // TODO: under construction function RegexpParser() { this.__init__.apply(this, arguments); } RegexpParser.prototype.RE_VERY_NOMAGIC = 1; @@ -5658,9 +5748,9 @@ RegexpParser.prototype.get_token_sq_char_class = function() { var r = this.reader.read_alpha(); if (this.reader.p(0) == ":" && this.reader.p(1) == "]") { this.reader.seek_cur(2); - var __c14 = class_names; - for (var __i14 = 0; __i14 < __c14.length; ++__i14) { - var name = __c14[__i14]; + var __c15 = class_names; + for (var __i15 = 0; __i15 < __c15.length; ++__i15) { + var name = __c15[__i15]; if (r == name) { return "[:" + name + ":]"; } @@ -5793,9 +5883,9 @@ RegexpParser.prototype.getoctchrs = function() { RegexpParser.prototype.gethexchrs = function(n) { var r = ""; - var __c15 = viml_range(n); - for (var __i15 = 0; __i15 < __c15.length; ++__i15) { - var i = __c15[__i15]; + var __c16 = viml_range(n); + for (var __i16 = 0; __i16 < __c16.length; ++__i16) { + var i = __c16[__i16]; var c = this.reader.peek(); if (!isxdigit(c)) { break; diff --git a/py/vimlfunc.py b/py/vimlfunc.py index 1e1b7331..12cf5ce7 100644 --- a/py/vimlfunc.py +++ b/py/vimlfunc.py @@ -90,6 +90,7 @@ class AttributeDict(dict): "^[0-9A-Fa-f][0-9A-Fa-f]$": "^[0-9A-Fa-f][0-9A-Fa-f]$", r"^\.[0-9A-Fa-f]$": r"^\.[0-9A-Fa-f]$", "^[0-9A-Fa-f][^0-9A-Fa-f]$": "^[0-9A-Fa-f][^0-9A-Fa-f]$", + "^[^a-z]\\S\\+$": "^[^a-z]\\S\\+$", } diff --git a/py/vimlparser.py b/py/vimlparser.py index 0ef7e1fb..0655fbb7 100644 --- a/py/vimlparser.py +++ b/py/vimlparser.py @@ -90,6 +90,7 @@ class AttributeDict(dict): "^[0-9A-Fa-f][0-9A-Fa-f]$": "^[0-9A-Fa-f][0-9A-Fa-f]$", r"^\.[0-9A-Fa-f]$": r"^\.[0-9A-Fa-f]$", "^[0-9A-Fa-f][^0-9A-Fa-f]$": "^[0-9A-Fa-f][^0-9A-Fa-f]$", + "^[^a-z]\\S\\+$": "^[^a-z]\\S\\+$", } @@ -307,6 +308,7 @@ def viml_stridx(a, b): NODE_BLOB = 93 NODE_CONST = 94 NODE_EVAL = 95 +NODE_HEREDOC = 96 TOKEN_EOF = 1 TOKEN_EOL = 2 TOKEN_SPACE = 3 @@ -375,6 +377,7 @@ def viml_stridx(a, b): TOKEN_BLOB = 66 TOKEN_LITCOPEN = 67 TOKEN_DOTDOT = 68 +TOKEN_HEREDOC = 69 MAX_FUNC_ARGS = 20 @@ -578,6 +581,7 @@ def ExArg(): # CURLYNAMEPART .value # CURLYNAMEEXPR .value # LAMBDA .rlist .left +# HEREDOC .rlist .op .body def Node(type): return AttributeDict({"type": type}) @@ -1473,6 +1477,35 @@ def parse_cmd_call(self): raise VimLParserException(Err("Not an function call", node.left.pos)) self.add_node(node) + def parse_heredoc(self): + node = Node(NODE_HEREDOC) + node.pos = self.ea.cmdpos + node.op = "" + node.rlist = [] + node.body = [] + while TRUE: + self.reader.skip_white() + key = self.reader.read_word() + if key == "": + break + if not islower(key[0]): + node.op = key + break + else: + viml_add(node.rlist, key) + if node.op == "": + raise VimLParserException(Err("E172: Missing marker", self.reader.getpos())) + self.parse_trail() + while TRUE: + if self.reader.peek() == "": + break + line = self.reader.getn(-1) + if line == node.op: + return node + viml_add(node.body, line) + self.reader.get() + raise VimLParserException(Err(viml_printf("E990: Missing end marker '%s'", node.op), self.reader.getpos())) + def parse_cmd_let(self): pos = self.reader.tell() self.reader.skip_white() @@ -1488,8 +1521,10 @@ def parse_cmd_let(self): # TODO check scriptversion? if s2 == "..": s2 = self.reader.peekn(3) + elif s2 == "=<": + s2 = self.reader.peekn(3) # :let {var-name} .. - if self.ends_excmds(s1) or s2 != "+=" and s2 != "-=" and s2 != ".=" and s2 != "..=" and s2 != "*=" and s2 != "/=" and s2 != "%=" and s1 != "=": + if self.ends_excmds(s1) or s2 != "+=" and s2 != "-=" and s2 != ".=" and s2 != "..=" and s2 != "*=" and s2 != "/=" and s2 != "%=" and s2 != "=<<" and s1 != "=": self.reader.seek_set(pos) self.parse_cmd_common() return @@ -1505,6 +1540,13 @@ def parse_cmd_let(self): if s2 == "+=" or s2 == "-=" or s2 == ".=" or s2 == "..=" or s2 == "*=" or s2 == "/=" or s2 == "%=": self.reader.getn(viml_len(s2)) node.op = s2 + elif s2 == "=<<": + self.reader.getn(viml_len(s2)) + self.reader.skip_white() + node.op = s2 + node.right = self.parse_heredoc() + self.add_node(node) + return elif s1 == "=": self.reader.getn(1) node.op = s1 @@ -3574,6 +3616,8 @@ def compile(self, node): return self.compile_curlynameexpr(node) elif node.type == NODE_LAMBDA: return self.compile_lambda(node) + elif node.type == NODE_HEREDOC: + return self.compile_heredoc(node) else: raise VimLParserException(viml_printf("Compiler: unknown node: %s", viml_string(node))) return NIL @@ -3957,10 +4001,34 @@ def compile_curlynamepart(self, node): def compile_curlynameexpr(self, node): return "{" + self.compile(node.value) + "}" + def escape_string(self, str): + m = AttributeDict({"\n": "\\n", "\t": "\\t", "\r": "\\r"}) + out = "\"" + for i in viml_range(viml_len(str)): + c = str[i] + if viml_has_key(m, c): + out += m[c] + else: + out += c + out += "\"" + return out + def compile_lambda(self, node): rlist = [self.compile(vval) for vval in node.rlist] return viml_printf("(lambda (%s) %s)", viml_join(rlist, " "), self.compile(node.left)) + def compile_heredoc(self, node): + if viml_empty(node.rlist): + rlist = "(list)" + else: + rlist = "(list " + viml_join([self.escape_string(vval) for vval in node.rlist], " ") + ")" + if viml_empty(node.body): + body = "(list)" + else: + body = "(list " + viml_join([self.escape_string(vval) for vval in node.body], " ") + ")" + op = self.escape_string(node.op) + return viml_printf("(heredoc %s %s %s)", rlist, op, body) + # TODO: under construction class RegexpParser: diff --git a/test/test_heredoc.ok b/test/test_heredoc.ok new file mode 100644 index 00000000..0de3f6bc --- /dev/null +++ b/test/test_heredoc.ok @@ -0,0 +1,3 @@ +(let =<< a (heredoc (list) "EOS" (list "hello" " world"))) +(let =<< a (heredoc (list "trim") "EOS" (list "\thello" "world"))) +(let =<< a (heredoc (list) "EOS" (list))) diff --git a/test/test_heredoc.vim b/test/test_heredoc.vim new file mode 100644 index 00000000..e92322b1 --- /dev/null +++ b/test/test_heredoc.vim @@ -0,0 +1,10 @@ +let a =<< EOS +hello + world +EOS +let a =<< trim EOS + hello +world +EOS +let a =<< EOS +EOS