diff --git a/plugin/UltiSnips/Lexer.py b/plugin/UltiSnips/Lexer.py new file mode 100644 index 0000000..ccd2755 --- /dev/null +++ b/plugin/UltiSnips/Lexer.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python +# encoding: utf-8 + +""" +Not really a Lexer in the classical sense, but code to hack Snippet Definitions +into Logical Units called Tokens. +""" + +import string +import re + +from Geometry import Position + +__all__ = [ + "tokenize", "EscapeCharToken", "TransformationToken", "TabStopToken", + "MirrorToken", "PythonCodeToken", "VimLCodeToken", "ShellCodeToken" +] + +# Helper Classes {{{ +class _TextIterator(object): + def __init__(self, text): + self._text = text + self._line = 0 + self._col = 0 + + self._idx = 0 + + def __iter__(self): + return self + + def next(self): + if self._idx >= len(self._text): + raise StopIteration + + rv = self._text[self._idx] + if self._text[self._idx] in ('\n', '\r\n'): + self._line += 1 + self._col = 0 + else: + self._col += 1 + self._idx += 1 + + return rv + + def peek(self, count = 1): + try: + return self._text[self._idx:self._idx + count] + except IndexError: + return None + + @property + def pos(self): + return Position(self._line, self._col) +# End: Helper Classes }}} +# Helper functions {{{ +def _parse_number(stream): + """ + Expects the stream to contain a number next, returns the number + without consuming any more bytes + """ + rv = "" + while stream.peek() and stream.peek() in string.digits: + rv += stream.next() + + return int(rv) + +def _parse_till_closing_brace(stream): + """ + Returns all chars till a non-escaped } is found. Other + non escaped { are taken into account and skipped over. + + Will also consume the closing }, but not return it + """ + rv = "" + in_braces = 1 + while True: + if EscapeCharToken.starts_here(stream, '{}'): + rv += stream.next() + stream.next() + else: + c = stream.next() + if c == '{': in_braces += 1 + elif c == '}': in_braces -= 1 + if in_braces == 0: break + rv += c + return rv + +def _parse_till_unescaped_char(stream, char): + """ + Returns all chars till a non-escaped `char` is found. + + Will also consume the closing `char`, but not return it + """ + rv = "" + while True: + if EscapeCharToken.starts_here(stream, char): + rv += stream.next() + stream.next() + else: + c = stream.next() + if c == char: break + rv += c + return rv +# End: Helper functions }}} + +# Tokens {{{ +class Token(object): + def __init__(self, gen, indent): + self.initial_text = "" + self.start = gen.pos + self._parse(gen, indent) + self.end = gen.pos + +class TabStopToken(Token): + CHECK = re.compile(r'^\${\d+[:}]') + + @classmethod + def starts_here(klass, stream): + return klass.CHECK.match(stream.peek(10)) != None + + def _parse(self, stream, indent): + stream.next() # $ + stream.next() # { + + self.no = _parse_number(stream) + + if stream.peek() is ":": + stream.next() + self.initial_text = _parse_till_closing_brace(stream) + + def __repr__(self): + return "TabStopToken(%r,%r,%r,%r)" % ( + self.start, self.end, self.no, self.initial_text + ) + +class TransformationToken(Token): + CHECK = re.compile(r'^\${\d+\/') + + @classmethod + def starts_here(klass, stream): + return klass.CHECK.match(stream.peek(10)) != None + + def _parse(self, stream, indent): + stream.next() # $ + stream.next() # { + + self.no = _parse_number(stream) + + stream.next() # / + + self.search = _parse_till_unescaped_char(stream, '/') + self.replace = _parse_till_unescaped_char(stream, '/') + self.options = _parse_till_closing_brace(stream) + + def __repr__(self): + return "TransformationToken(%r,%r,%r,%r,%r)" % ( + self.start, self.end, self.no, self.search, self.replace + ) + +class MirrorToken(Token): + CHECK = re.compile(r'^\$\d+') + + @classmethod + def starts_here(klass, stream): + return klass.CHECK.match(stream.peek(10)) != None + + def _parse(self, stream, indent): + stream.next() # $ + self.no = _parse_number(stream) + + def __repr__(self): + return "MirrorToken(%r,%r,%r)" % ( + self.start, self.end, self.no + ) + +class EscapeCharToken(Token): + @classmethod + def starts_here(klass, stream, chars = '{}\$`'): + cs = stream.peek(2) + if len(cs) == 2 and cs[0] == '\\' and cs[1] in chars: + return True + + def _parse(self, stream, indent): + stream.next() # \ + self.initial_text = stream.next() + + def __repr__(self): + return "EscapeCharToken(%r,%r,%r)" % ( + self.start, self.end, self.initial_text + ) + +class ShellCodeToken(Token): + @classmethod + def starts_here(klass, stream): + return stream.peek(1) == '`' + + def _parse(self, stream, indent): + stream.next() # ` + self.code = _parse_till_unescaped_char(stream, '`') + + def __repr__(self): + return "ShellCodeToken(%r,%r,%r)" % ( + self.start, self.end, self.code + ) + +class PythonCodeToken(Token): + CHECK = re.compile(r'^`!p\s') + + @classmethod + def starts_here(klass, stream): + return klass.CHECK.match(stream.peek(4)) is not None + + def _parse(self, stream, indent): + for i in range(3): + stream.next() # `!p + if stream.peek() in '\t ': + stream.next() + + code = _parse_till_unescaped_char(stream, '`') + + # Strip the indent if any + if len(indent): + lines = code.splitlines() + self.code = lines[0] + '\n' + self.code += '\n'.join([l[len(indent):] + for l in lines[1:]]) + else: + self.code = code + self.indent = indent + + def __repr__(self): + return "PythonCodeToken(%r,%r,%r)" % ( + self.start, self.end, self.code + ) + +class VimLCodeToken(Token): + CHECK = re.compile(r'^`!v\s') + + @classmethod + def starts_here(klass, stream): + return klass.CHECK.match(stream.peek(4)) is not None + + def _parse(self, stream, indent): + for i in range(4): + stream.next() # `!v + self.code = _parse_till_unescaped_char(stream, '`') + + def __repr__(self): + return "VimLCodeToken(%r,%r,%r)" % ( + self.start, self.end, self.code + ) +# End: Tokens }}} + +__ALLOWED_TOKENS = [ + EscapeCharToken, TransformationToken, TabStopToken, MirrorToken, + PythonCodeToken, VimLCodeToken, ShellCodeToken +] +def tokenize(text, indent): + stream = _TextIterator(text) + + while True: + done_something = False + for t in __ALLOWED_TOKENS: + if t.starts_here(stream): + yield t(stream, indent) + done_something = True + break + if not done_something: + stream.next() + + diff --git a/plugin/UltiSnips/TextObjects.py b/plugin/UltiSnips/TextObjects.py index d7d2f1f..05a7558 100644 --- a/plugin/UltiSnips/TextObjects.py +++ b/plugin/UltiSnips/TextObjects.py @@ -10,6 +10,8 @@ import vim from UltiSnips.Util import IndentUtil from UltiSnips.Buffer import TextBuffer from UltiSnips.Geometry import Span, Position +from UltiSnips.Lexer import tokenize, EscapeCharToken, TransformationToken, \ + TabStopToken, MirrorToken, PythonCodeToken, VimLCodeToken, ShellCodeToken __all__ = [ "Mirror", "Transformation", "SnippetInstance", "StartMarker" ] @@ -109,315 +111,95 @@ class _CleverReplace(object): return self._unescape(tv.decode("string-escape")) class _TOParser(object): - # A simple tabstop with default value - _TABSTOP = re.compile(r'''(? %s)" % (self._start, self._end) class ShellCode(TextObject): - def __init__(self, parent, start, end, code): - - code = code.replace("\\`", "`") + def __init__(self, parent, token): + code = token.code.replace("\\`", "`") # Write the code to a temporary file handle, path = tempfile.mkstemp(text=True) @@ -678,16 +457,17 @@ class ShellCode(TextObject): os.unlink(path) - TextObject.__init__(self, parent, start, end, output) + token.initial_text = output + TextObject.__init__(self, parent, token) def __repr__(self): return "ShellCode(%s -> %s)" % (self._start, self._end) class VimLCode(TextObject): - def __init__(self, parent, start, end, code): - self._code = code.replace("\\`", "`").strip() + def __init__(self, parent, token): + self._code = token.code.replace("\\`", "`").strip() - TextObject.__init__(self, parent, start, end, "") + TextObject.__init__(self, parent, token) def _do_update(self): self.current_text = str(vim.eval(self._code)) @@ -844,9 +624,9 @@ class SnippetUtil(object): class PythonCode(TextObject): - def __init__(self, parent, start, end, code, indent=""): + def __init__(self, parent, token): - code = code.replace("\\`", "`") + code = token.code.replace("\\`", "`") # Find our containing snippet for snippet local data snippet = parent @@ -855,7 +635,7 @@ class PythonCode(TextObject): snippet = snippet._parent except AttributeError: snippet = None - self._snip = SnippetUtil(indent) + self._snip = SnippetUtil(token.indent) self._locals = snippet.locals self._globals = {} @@ -865,7 +645,7 @@ class PythonCode(TextObject): # Add Some convenience to the code self._code = "import re, os, vim, string, random\n" + code - TextObject.__init__(self, parent, start, end, "") + TextObject.__init__(self, parent, token) def _do_update(self): @@ -903,9 +683,13 @@ class TabStop(TextObject): This is the most important TextObject. A TabStop is were the cursor comes to rest when the user taps through the Snippet. """ - def __init__(self, no, parent, start, end, default_text = ""): - TextObject.__init__(self, parent, start, end, default_text) - self._no = no + def __init__(self, parent, token, start = None, end = None): + if start is not None: + self._no = token + TextObject.__init__(self, parent, start, end) + else: + TextObject.__init__(self, parent, token) + self._no = token.no def no(self): return self._no @@ -952,7 +736,7 @@ class SnippetInstance(TextObject): col -= self.start.col start = Position(delta.line, col) end = Position(delta.line, col) - ts = TabStop(0, self, start, end, "") + ts = TabStop(self, 0, start, end) self._add_tabstop(0,ts) self.update() @@ -1000,5 +784,3 @@ class SnippetInstance(TextObject): return ts return self._tabstops[self._cts] - - diff --git a/test.py b/test.py index effd6bb..803d3a8 100755 --- a/test.py +++ b/test.py @@ -443,6 +443,18 @@ class TabStop_EscapingCharsDollars(_VimTest): snippets = ("test", r"snip \$0 $$0 end") keys = "test" + EX + "hi" wanted = "snip $0 $hi end" +class TabStop_EscapingCharsDollars1(_VimTest): + snippets = ("test", r"a\${1:literal}") + keys = "test" + EX + wanted = "a${1:literal}" +class TabStop_EscapingCharsDollars_BeginningOfLine(_VimTest): + snippets = ("test", "\n\\${1:literal}") + keys = "test" + EX + wanted = "\n${1:literal}" +class TabStop_EscapingCharsDollars_BeginningOfDefinitionText(_VimTest): + snippets = ("test", "\\${1:literal}") + keys = "test" + EX + wanted = "${1:literal}" class TabStop_EscapingChars_Backslash(_VimTest): snippets = ("test", r"This \ is a backslash!") keys = "test" + EX