Skip to content

Rewrite token emitting functions #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 18, 2014
Merged

Rewrite token emitting functions #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
97b3c64
Fix incorrect syntax of defrecord test.
mattdeboard Aug 18, 2014
2d570b1
Rewrite of forward/backward token functionality.
mattdeboard Aug 18, 2014
ae60c35
Remove unused rules & warehoused code.
mattdeboard Aug 18, 2014
1f605e3
Remove logic to emit ';' after a hanging comma.
mattdeboard Aug 18, 2014
ae86f70
Remove unnecessary regexps.
mattdeboard Aug 18, 2014
43f0a06
Rearrange code a bit.
mattdeboard Aug 18, 2014
aecec45
Make sure comments are handled correctly when pressing return.
mattdeboard Aug 18, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion elixir-mode-tests.el
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
,@body))

(load "test/elixir-mode-indentation-tests.el")
(load "test/elixir-mode-tokenizer-hl-tests.el")
(load "test/elixir-mode-font-tests.el")

(provide 'elixir-mode-tests)
Expand Down
299 changes: 82 additions & 217 deletions elixir-smie.el
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,215 +34,90 @@
table)
"Elixir mode syntax table.")

(defconst elixir-smie-grammar
(smie-prec2->grammar
(smie-merge-prec2s
(smie-bnf->prec2
'((id)
(statements (statement)
(statement ";" statements))
(statement ("def" non-block-expr "do" statements "end")
(non-block-expr "fn" match-statement "end")
(non-block-expr "do" statements "end")
("if" non-block-expr "do" statements "else" statements "end")
("if" non-block-expr "do" statements "end")
("if" non-block-expr "COMMA" "do:" non-block-expr)
("if" non-block-expr "COMMA"
"do:" non-block-expr "COMMA"
"else:" non-block-expr)
("try" "do" statements "after" statements "end")
("try" "do" statements "catch" match-statements "end")
("try" "do" statements "end")
("case" non-block-expr "do" match-statements "end"))
(non-block-expr (non-block-expr "OP" non-block-expr)
(non-block-expr "COMMA" non-block-expr)
("(" non-block-expr ")")
("{" non-block-expr "}")
("[" non-block-expr "]")
("STRING"))
(match-statements (match-statement "MATCH-STATEMENT-DELIMITER"
match-statements)
(match-statement))
(match-statement (non-block-expr "->" statements)))
'((assoc "if" "do:" "else:")
(assoc "COMMA")
(left "OP")))

(smie-precs->prec2
'((left "||")
(left "&&")
(nonassoc "=~" "===" "!==" "==" "!=" "<=" ">=" "<" ">")
(left "+" "-" "<<<" ">>>" "^^^" "~~~" "&&&" "|||")
(left "*" "/"))))))

(defvar elixir-smie--operator-regexp
(regexp-opt '("<<<" ">>>" "^^^" "~~~" "&&&" "|||" "===" "!==" "==" "!=" "<="
">=" "<" ">" "&&" "||" "<>" "++" "--" "//"
"/>" "=~" "|>" "->")))

(defvar elixir-smie-indent-basic 2)

(defmacro elixir-smie-debug (message &rest format-args)
`(progn
(when elixir-smie-verbose-p
(message (format ,message ,@format-args)))
nil))

(progn
(setq elixir-syntax-class-names nil)

(defmacro elixir-smie-define-regexp-opt (name &rest table)
`(elixir-smie-define-regexp ,name (regexp-opt (list ,@table))))

(defmacro elixir-smie-define-regexp (name regexp &optional flag)
(let ((regex-name (intern (format "elixir-smie-%s" name))))
`(progn
(defconst ,regex-name
,regexp)
(pushnew `(,',regex-name . ,(upcase (symbol-name ',name))) elixir-syntax-class-names))))

(elixir-smie-define-regexp-opt op "&&" "||" "!")
(elixir-smie-define-regexp dot "\\.")
(elixir-smie-define-regexp comma ",")
(elixir-smie-define-regexp -> "->")
(elixir-smie-define-regexp << "<<")
(elixir-smie-define-regexp >> ">>")
(elixir-smie-define-regexp-opt parens "(" ")" "{" "}" "[" "]" "<<" ">>"))

(defconst elixir-smie-block-intro-keywords
'(do else catch after rescue -> OP)
"Keywords in which newlines cause confusion for the parser.")

(defun elixir-skip-comment-backward ()
"Skip backwards over all whitespace and comments.

Return non-nil if any line breaks were skipped."
(let ((start-line-no (line-number-at-pos (point))))
(forward-comment (- (point)))
(/= start-line-no (line-number-at-pos (point)))))

(defun elixir-skip-comment-forward ()
"Skip forward over any whitespace and comments.

Return non-nil if any line breaks were skipped."
(let ((start-line-no (line-number-at-pos (point))))
(forward-comment (buffer-size))
(/= start-line-no (line-number-at-pos (point)))))

(defun elixir-smie-next-token-no-lookaround (forwardp)
(block elixir-smie-next-token-no-lookaround
;; First, skip comments; but if any comments / newlines were
;; skipped, the upper level needs to check if they were significant:
(when (if forwardp
(elixir-skip-comment-forward)
(elixir-skip-comment-backward))
(return-from elixir-smie-next-token-no-lookaround "\n"))
(let* ((found-token-class (find-if
(lambda (class-def)
(let ((regex (symbol-value (car class-def))))
(if forwardp
(looking-at regex)
(looking-back regex nil t))))
elixir-syntax-class-names))
(maybe-token
(let ((current-char (if forwardp
(following-char)
(preceding-char))))
(cond ((member current-char
'(?\n ?\;))
(if forwardp
(forward-comment (point-max))
(forward-comment (- (point))))
(string current-char))
(found-token-class
(goto-char (if forwardp
(match-end 0)
(match-beginning 0)))
(if (string= "PARENS" (cdr found-token-class))
(buffer-substring-no-properties (match-beginning 0) (match-end 0))
(cdr found-token-class)))
((when (= ?\" (char-syntax (if forwardp
(following-char)
(preceding-char))))
(if forwardp
(forward-sexp)
(backward-sexp))
"STRING"))))))
(or maybe-token
(downcase
(buffer-substring-no-properties
(point)
(if forwardp
(progn (skip-syntax-forward "'w_")
(point))
(progn (skip-syntax-backward "'w_")
(point)))))))))
(defun elixir-smie--at-dot-call ()
(and (eq ?w (char-syntax (following-char)))
(eq (char-before) ?.)
(not (eq (char-before (1- (point))) ?.))))

(defun elixir-smie-next-token (forwardp)
(block elixir-smie-next-token
(let ((current-token (elixir-smie-next-token-no-lookaround forwardp)))
(when (string= "\n" current-token)
;; This is a newline; if the previous token isn't an OP2, this
;; means the line end marks the end of a statement & we get to
;; scan forward until there's a non-newline token; otherwise,
;; make this line ending something that probably ends the
;; statement (but see below).
(if (save-excursion
(block nil
(let ((token (elixir-smie-next-token-no-lookaround nil)))
(while (and (not (= (point) (point-min)))
(string= "\n" token))
(setq token (elixir-smie-next-token-no-lookaround nil)))
(when (member (intern token) elixir-smie-block-intro-keywords)
(return t)))))
;; it's a continuation line, return the next token after the newline:
(return-from elixir-smie-next-token (elixir-smie-next-token forwardp))
(setq current-token ";")))

;; When reading match statements (the ones with expr -> statements),
;; we need to drop non-; delimiters so the parser knows when a
;; match statement ends and another begins, so scan around point to
;; see if there are any -> within the current block's scope.

;; If the current token is a ";", scan forward to see if the current
;; potential statement contains a "->". If so, scan back to find a
;; "do". If there is a -> there, emit a match-statement-delimiter
;; instead of the ";".
(if (and (string= ";" current-token)
;; Scan ahead:
(let ((level 0)
token)
(save-excursion
(block nil
(while
(and
;; Cursor is not at the end of the buffer...
(not (= (point) (point-max)))
;; ...and the current token is not an empty string...
(not (string= "" token))
;; ...nor a newline nor a semicolon.
(not (or (string= "\n" token) (string= ";" token))))
(setq token (elixir-smie-next-token-no-lookaround t))
;; If we're at the top level and the token is "->",
;; return t
(cond ((and (= level 0) (string= "->" token))
(return t))
;; If token is "do" or "fn", increment level
((find token '("do" "fn") :test 'string=)
(incf level))
;; If token is "end", decrement level
((string= token "end")
(decf level)))))))
;; Scan behind:
(let (token)
(save-excursion
(block nil
(while
(and
;; Cursor is not at the beginning of buffer...
(not (= (point) (point-min)))
;; ...and token is neither empty string, nor "do"/"fn"
(not (string= "" token))
(not (string= "do" token))
(not (string= "fn" token)))
(setq token (elixir-smie-next-token-no-lookaround nil))
(when (string= "->" token)
(return t)))
(when (string= token "do") t)))))
"MATCH-STATEMENT-DELIMITER"
current-token))))
(defun elixir-smie--implicit-semi-p ()
(not (or (memq (char-before) '(?\{ ?\[))
(looking-back elixir-smie--operator-regexp (- (point) 3) t))))

(defun elixir-smie-forward-token ()
(elixir-smie-next-token t))
(cond
((and (looking-at "[\n#]") (elixir-smie--implicit-semi-p))
(if (eolp) (forward-char 1) (forward-comment 1))
";")
((looking-at elixir-smie--operator-regexp)
(goto-char (match-end 0))
"OP")
(t (smie-default-forward-token))))

(defun elixir-smie-backward-token ()
(elixir-smie-next-token nil))

(defconst elixir-smie-grammar
(smie-prec2->grammar
(smie-bnf->prec2
'((id)
(statements (statement)
(statement ";" statements))
(statement ("def" non-block-expr "do" statements "end")
(non-block-expr "fn" match-statement "end")
(non-block-expr "do" statements "end")
("if" non-block-expr "do" statements "else" statements "end")
("if" non-block-expr "do" statements "end")
("if" non-block-expr "COMMA" "do:" non-block-expr)
("if" non-block-expr "COMMA"
"do:" non-block-expr "COMMA"
"else:" non-block-expr)
("try" "do" statements "after" statements "end")
("try" "do" statements "catch" match-statements "end")
("try" "do" statements "end")
("case" non-block-expr "do" match-statements "end"))
(non-block-expr (non-block-expr "OP" non-block-expr)
(non-block-expr "COMMA" non-block-expr)
("(" statements ")")
("{" statements "}")
("[" statements "]")
("STRING"))
(match-statements (match-statement "MATCH-STATEMENT-DELIMITER" match-statements)
(match-statement))
(match-statement (non-block-expr "->" statements)))
'((assoc "if" "do:" "else:")
(assoc "COMMA")
(left "OP")))))

(defvar elixir-smie-indent-basic 2)
(let ((pos (point)))
(forward-comment (- (point)))
(cond
((and (> pos (line-end-position))
(elixir-smie--implicit-semi-p))
";")
((looking-back elixir-smie--operator-regexp (- (point) 3) t)
(goto-char (match-beginning 0))
"OP")
(t (smie-default-backward-token)))))

(defun verbose-elixir-smie-rules (kind token)
(let ((value (elixir-smie-rules kind token)))
Expand All @@ -256,35 +131,25 @@ Return non-nil if any line breaks were skipped."

(defun elixir-smie-rules (kind token)
(pcase (cons kind token)
(`(:after . "STRING")
(if (smie-rule-prev-p "do:")
(smie-rule-parent 0)
nil))
(`(:elem . basic)
(if (smie-rule-hanging-p)
0
elixir-smie-indent-basic))
(`(:after . "OP")
(unless (smie-rule-sibling-p)
elixir-smie-indent-basic))
(cond
((smie-rule-sibling-p) nil)
((smie-rule-hanging-p) (smie-rule-parent elixir-smie-indent-basic))
(t elixir-smie-indent-basic)))
(`(:before . "def") elixir-smie-indent-basic)
;; If the parent token of `->' is `fn', then we want to align to the
;; parent, and offset by `elixir-smie-indent-basic'. Otherwise, indent
;; normally. This helps us work with/indent anonymous function blocks
;; correctly.
(`(:after . "->")
(when (smie-rule-hanging-p)
(if (smie-rule-parent-p "fn")
(smie-rule-parent elixir-smie-indent-basic)
elixir-smie-indent-basic)))
(`(:after . "do")
elixir-smie-indent-basic)
(`(:list-intro . ,(or `"do" `";")) t)
(`(:before . ";")
(cond
((smie-rule-parent-p "after" "catch" "def" "defmodule" "defp" "do" "else"
"fn" "if" "rescue" "try" "unless")
(smie-rule-parent elixir-smie-indent-basic))))
(`(:after . ";")
(if (smie-rule-parent-p "if")
(smie-rule-parent 0)))))


(define-minor-mode elixir-smie-mode
"SMIE-based indentation and syntax for Elixir"
nil nil nil nil
Expand Down
28 changes: 19 additions & 9 deletions test/elixir-mode-indentation-tests.el
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,26 +176,36 @@ end

(elixir-def-indentation-test indents-records-correctly ()
"
defrecord Money, [:currency_unit, :amount] do
foo
defmodule MyModule do
require Record
Record.defrecord :money, [:currency_unit, :amount]

Record.defrecord :animal, [:species, :name]
end
"
"
defrecord Money, [:currency_unit, :amount] do
foo
defmodule MyModule do
require Record
Record.defrecord :money, [:currency_unit, :amount]

Record.defrecord :animal, [:species, :name]
end
")

(elixir-def-indentation-test indents-continuation-lines ()
"
has_something(x) &&
has_something(y) ||
has_something(z)
"
"
def foo do
has_something(x) &&
has_something(y) ||
has_something(z)
end
"
"
def foo do
has_something(x) &&
has_something(y) ||
has_something(z)
end
")

(elixir-def-indentation-test indents-continuation-lines-with-comments/1
Expand Down
Loading