Refactored how snippet files are parsed.

pylintrc

@@ -141,7 +141,7 @@ indent-string='    '
 [MISCELLANEOUS]
 
 # List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
+notes=TODO
 
 
 [SIMILARITIES]

pythonx/UltiSnips/__init__.py

@@ -14,8 +14,8 @@ from UltiSnips._diff import diff, guess_edit
 from UltiSnips.compatibility import as_unicode
 from UltiSnips.position import Position
 from UltiSnips.snippet import Snippet
+from UltiSnips.snippet_definitions import parse_snippets_file
 from UltiSnips.snippet_dictionary import SnippetDictionary
-from UltiSnips.snippets_file_parser import SnippetsFileParser
 from UltiSnips.vim_state import VimState, VisualContentPreserver
 import UltiSnips._vim as _vim
 
@@ -299,24 +299,6 @@ class SnippetManager(object):
         else:
             return False
 
-    @err_to_scratch_buffer
-    def clear_snippets(self, triggers=None, ft="all"):
-        """Forget all snippets for the given 'ft'. If 'triggers' is given only
-        forget those with the given trigger."""
-        if triggers is None:
-            triggers = []
-        if ft in self._snippets:
-            self._snippets[ft].clear_snippets(triggers)
-
-    @err_to_scratch_buffer
-    def add_extending_info(self, ft, parents):
-        """Add the list of 'parents' as being extended by the 'ft'."""
-        sd = self._snippets[ft]
-        for parent in parents:
-            if parent in sd.extends:
-                continue
-            sd.extends.append(parent)
-
     @err_to_scratch_buffer
     def cursor_moved(self):
         """Called whenever the cursor moved."""
@@ -389,8 +371,10 @@ class SnippetManager(object):
             self._current_snippet_is_done()
         self._reinit()
 
-    # TODO(sirver): This is only used by SnippetsFileParser
+    ###################################
-    def report_error(self, msg):
+    # Private/Protect Functions Below #
+    ###################################
+    def _report_error(self, msg):
         """Shows 'msg' as error to the user."""
         msg = _vim.escape("UltiSnips: " + msg)
         if self._test_error:
@@ -406,9 +390,14 @@ class SnippetManager(object):
         else:
             _vim.command("echoerr %s" % msg)
 
-    ###################################
+    def _add_extending_info(self, ft, parents):
-    # Private/Protect Functions Below #
+        """Add the list of 'parents' as being extended by the 'ft'."""
-    ###################################
+        sd = self._snippets[ft]
+        for parent in parents:
+            if parent in sd.extends:
+                continue
+            sd.extends.append(parent)
+
     def _reinit(self):
         """Resets transient state."""
         self._ctab = None
@@ -602,9 +591,29 @@ class SnippetManager(object):
 
     def _parse_snippets(self, ft, filename, file_data=None):
         """Parse the file 'filename' for the given 'ft' and watch it for
-        changes in the future."""
+        changes in the future. 'file_data' can be injected in tests."""
         self._snippets[ft].addfile(filename)
-        SnippetsFileParser(ft, filename, self, file_data).parse()
+        if file_data is None:
+            file_data = file_data.read()
+        for event, data in parse_snippets_file(file_data):
+            if event == "error":
+                msg, line_index = data
+                filename = _vim.eval("""fnamemodify(%s, ":~:.")""" %
+                        _vim.escape(filename))
+                self._report_error("%s in %s(%d)" % (msg, filename, line_index))
+                break
+            elif event == "clearsnippets":
+                triggers, = data
+                self._snippets[ft].clear_snippets(triggers)
+            elif event == "extends":
+                filetypes, = data
+                self._add_extending_info(ft, filetypes)
+            elif event == "snippet":
+                trigger, value, descr, opts, globals = data
+                self.add_snippet(trigger, value, descr,
+                        opts, ft, globals, filename)
+            else:
+                assert False, "Unhandled %s: %r" % (event, data)
 
     @property
     def primary_filetype(self):

pythonx/UltiSnips/snippet_definitions.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+"""Parsing of snippet files."""
+
+from collections import defaultdict
+
+class _LineIterator(object):
+    """Convenience class that keeps track of line numbers."""
+
+    def __init__(self, text):
+        self._line_index = None
+        self._lines = enumerate(text.splitlines(True), 1)
+
+    def __iter__(self):
+        return self
+
+    @property
+    def line_index(self):
+        """The 1 based line index in the current file."""
+        return self._line_index
+
+    def next(self):
+        """Returns the next line."""
+        self._line_index, line = next(self._lines)
+        return line
+
+def _handle_snippet_or_global(line, lines, globals):
+    """Parses the snippet that begins at the current line."""
+
+    descr = ""
+    opts = ""
+
+    # Ensure this is a snippet
+    snip = line.split()[0]
+
+    # Get and strip options if they exist
+    remain = line[len(snip):].strip()
+    words = remain.split()
+    if len(words) > 2:
+        # second to last word ends with a quote
+        if '"' not in words[-1] and words[-2][-1] == '"':
+            opts = words[-1]
+            remain = remain[:-len(opts) - 1].rstrip()
+
+    # Get and strip description if it exists
+    remain = remain.strip()
+    if len(remain.split()) > 1 and remain[-1] == '"':
+        left = remain[:-1].rfind('"')
+        if left != -1 and left != 0:
+            descr, remain = remain[left:], remain[:left]
+
+    # The rest is the trigger
+    trig = remain.strip()
+    if len(trig.split()) > 1 or "r" in opts:
+        if trig[0] != trig[-1]:
+            return "error", ("Invalid multiword trigger: '%s'" % trig,
+                    lines.line_index)
+        trig = trig[1:-1]
+    end = "end" + snip
+    content = ""
+
+    found_end = False
+    for line in lines:
+        if line.rstrip() == end:
+            content = content[:-1]  # Chomp the last newline
+            found_end = True
+            break
+        content += line
+
+    if not found_end:
+        return "error", ("Missing 'endsnippet' for %r" % trig, lines.line_index)
+
+    if snip == "global":
+        globals[trig].append(content)
+    elif snip == "snippet":
+        return "snippet", (trig, content, descr, opts, globals)
+    else:
+        return "error", ("Invalid snippet type: '%s'" % snip, lines.line_index)
+
+def _head_tail(line):
+    """Returns the first word in 'line' and the rest of 'line' or None if the
+    line is too short."""
+    generator = (t.strip() for t in line.split(None, 1))
+    head = next(generator).strip()
+    tail = ''
+    try:
+        tail = next(generator).strip()
+    except StopIteration:
+        pass
+    return head, tail
+
+
+def parse_snippets_file(data):
+    """Parse 'data' assuming it is a snippet file. Yields events in the
+    file."""
+
+    globals = defaultdict(list)
+    lines = _LineIterator(data)
+    for line in lines:
+        if not line.strip():
+            continue
+
+        head, tail = _head_tail(line)
+        if head == "extends":
+            if tail:
+                yield "extends", ([p.strip() for p in tail.split(',')],)
+            else:
+                yield "error", ("'extends' without file types",
+                        lines.line_index)
+        elif head in ("snippet", "global"):
+            snippet = _handle_snippet_or_global(line, lines, globals)
+            if snippet is not None:
+                yield snippet
+        elif head == "clearsnippets":
+            yield "clearsnippets", (tail.split(),)
+        elif head and not head.startswith('#'):
+            yield "error", ("Invalid line %r" % line.rstrip(), lines.line_index)

pythonx/UltiSnips/snippets_file_parser.py

@@ -1,134 +0,0 @@
-#!/usr/bin/env python
-# encoding: utf-8
-
-"""Parsing of snippet files."""
-
-import re
-import UltiSnips._vim as _vim
-
-# TODO(sirver): This could just as well be a function. Also the
-# interface should change to a stream of events - so that it does
-# not need knowledge of SnippetManager.
-class SnippetsFileParser(object):
-    """Does the actual parsing."""
-
-    def __init__(self, ft, fn, snip_manager, file_data=None):
-        """Parser 'fn' as filetype 'ft'."""
-        self._sm = snip_manager
-        self._ft = ft
-        self._fn = fn
-        self._globals = {}
-        if file_data is None:
-            self._lines = open(fn).readlines()
-        else:
-            self._lines = file_data.splitlines(True)
-        self._idx = 0
-
-    def _error(self, msg):
-        """Reports 'msg' as an error."""
-        fn = _vim.eval("""fnamemodify(%s, ":~:.")""" % _vim.escape(self._fn))
-        self._sm.report_error("%s in %s(%d)" % (msg, fn, self._idx + 1))
-
-    def _line(self):
-        """The current line or the empty string."""
-        return self._lines[self._idx] if self._idx < len(self._lines) else ""
-
-    def _line_head_tail(self):
-        """Returns (first word, rest) of the current line."""
-        parts = re.split(r"\s+", self._line().rstrip(), maxsplit=1)
-        parts.append('')
-        return parts[:2]
-
-    def _goto_next_line(self):
-        """Advances to and returns the next line."""
-        self._idx += 1
-        return self._line()
-
-    def _parse_first(self, line):
-        """Parses the first line of the snippet definition. Returns the
-        snippet type, trigger, description, and options in a tuple in that
-        order.
-        """
-        cdescr = ""
-        coptions = ""
-        cs = ""
-
-        # Ensure this is a snippet
-        snip = line.split()[0]
-
-        # Get and strip options if they exist
-        remain = line[len(snip):].strip()
-        words = remain.split()
-        if len(words) > 2:
-            # second to last word ends with a quote
-            if '"' not in words[-1] and words[-2][-1] == '"':
-                coptions = words[-1]
-                remain = remain[:-len(coptions) - 1].rstrip()
-
-        # Get and strip description if it exists
-        remain = remain.strip()
-        if len(remain.split()) > 1 and remain[-1] == '"':
-            left = remain[:-1].rfind('"')
-            if left != -1 and left != 0:
-                cdescr, remain = remain[left:], remain[:left]
-
-        # The rest is the trigger
-        cs = remain.strip()
-        if len(cs.split()) > 1 or "r" in coptions:
-            if cs[0] != cs[-1]:
-                self._error("Invalid multiword trigger: '%s'" % cs)
-                cs = ""
-            else:
-                cs = cs[1:-1]
-        return (snip, cs, cdescr, coptions)
-
-    def _parse_snippet(self):
-        """Parses the snippet that begins at the current line."""
-        line = self._line()
-
-        (snip, trig, desc, opts) = self._parse_first(line)
-        end = "end" + snip
-        cv = ""
-
-        while self._goto_next_line():
-            line = self._line()
-            if line.rstrip() == end:
-                cv = cv[:-1] # Chop the last newline
-                break
-            cv += line
-        else:
-            self._error("Missing 'endsnippet' for %r" % trig)
-            return None
-
-        if not trig:
-            # there was an error
-            return None
-        elif snip == "global":
-            # add snippet contents to file globals
-            if trig not in self._globals:
-                self._globals[trig] = []
-            self._globals[trig].append(cv)
-        elif snip == "snippet":
-            self._sm.add_snippet(
-                trig, cv, desc, opts, self._ft, self._globals, fn=self._fn)
-        else:
-            self._error("Invalid snippet type: '%s'" % snip)
-
-    def parse(self):
-        """Parses the given file."""
-        while self._line():
-            head, tail = self._line_head_tail()
-            if head == "extends":
-                if tail:
-                    self._sm.add_extending_info(self._ft,
-                        [p.strip() for p in tail.split(',')])
-                else:
-                    self._error("'extends' without file types")
-            elif head in ("snippet", "global"):
-                self._parse_snippet()
-            elif head == "clearsnippets":
-                self._sm.clear_snippets(tail.split(), self._ft)
-            elif head and not head.startswith('#'):
-                self._error("Invalid line %r" % self._line().rstrip())
-                break
-            self._goto_next_line()

test.py

@@ -358,7 +358,7 @@ class ParseSnippets_MissingEndSnippet(_PS_Base):
     keys = "testsnip" + EX
     wanted = "testsnip" + EX
     expected_error = dedent("""
-        UltiSnips: Missing 'endsnippet' for 'testsnip' in test_file(5)
+        UltiSnips: Missing 'endsnippet' for 'testsnip' in test_file(4)
         """).strip()
 
 class ParseSnippets_UnknownDirective(_PS_Base):