diff --git a/test/smoke_test.vader b/test/smoke_test.vader
index 6fbdc121..1cbf512d 100644
--- a/test/smoke_test.vader
+++ b/test/smoke_test.vader
@@ -35,6 +35,7 @@ After:
 
   unlet! g:i
   unlet! g:results
+  unlet! g:item
   unlet! g:expected_results
 
   delfunction TestCallback
@@ -50,10 +51,7 @@ Given foobar (Some imaginary filetype):
 Execute(Linters should run with the default options):
   AssertEqual 'foobar', &filetype
 
-  call ale#Lint()
-  call ale#engine#WaitForJobs(2000)
-
-  AssertEqual [{
+  let g:expected_results = [{
   \   'bufnr': bufnr('%'),
   \   'lnum': 2,
   \   'vcol': 0,
@@ -63,7 +61,28 @@ Execute(Linters should run with the default options):
   \   'nr': -1,
   \   'pattern': '',
   \   'valid': 1,
-  \ }], getloclist(0)
+  \ }]
+
+  " Try the test a few times over in NeoVim 0.3 or Windows,
+  " where tests fail randomly.
+  for g:i in range(has('nvim-0.3') || has('win32') ? 5 : 1)
+    call ale#Lint()
+    call ale#engine#WaitForJobs(2000)
+
+    let g:results = getloclist(0)
+
+    for g:item in g:results
+      if has_key(g:item, 'module')
+        call remove(g:item, 'module')
+      endif
+    endfor
+
+    if g:results == g:expected_results
+      break
+    endif
+  endfor
+
+  AssertEqual g:expected_results, g:results
 
 Execute(Linters should run in PowerShell too):
   if has('win32')
@@ -159,6 +178,12 @@ Execute(Previous errors should be removed when linters change):
 
     let g:results = getloclist(0)
 
+    for g:item in g:results
+      if has_key(g:item, 'module')
+        call remove(g:item, 'module')
+      endif
+    endfor
+
     if g:results == g:expected_results
       break
     endif
diff --git a/test/test_errors_removed_after_filetype_changed.vader b/test/test_errors_removed_after_filetype_changed.vader
index 92d248d0..afd882b1 100644
--- a/test/test_errors_removed_after_filetype_changed.vader
+++ b/test/test_errors_removed_after_filetype_changed.vader
@@ -1,8 +1,23 @@
 Before:
+  Save &filetype
+  Save g:ale_buffer_info
+  Save g:ale_echo_cursor
   Save g:ale_run_synchronously
+  Save g:ale_run_synchronously
+  Save g:ale_set_highlights
+  Save g:ale_set_loclist
+  Save g:ale_set_quickfix
+  Save g:ale_set_signs
+
+  " Enable only the one feature we need.
+  let g:ale_set_signs = 0
+  let g:ale_set_quickfix = 0
+  let g:ale_set_loclist = 1
+  let g:ale_set_highlights = 0
+  let g:ale_echo_cursor = 0
 
-  let b:old_filetype = &filetype
   let g:ale_run_synchronously = 1
+  call setloclist(0, [])
 
   noautocmd let &filetype = 'foobar'
 
@@ -10,6 +25,7 @@ Before:
     return [{'text': 'x', 'lnum': 1}]
   endfunction
 
+  call ale#linter#PreventLoading('foobar')
   call ale#linter#Define('foobar', {
   \ 'name': 'buffer_linter',
   \ 'callback': 'TestCallback',
@@ -17,7 +33,7 @@ Before:
   \ 'command': 'true',
   \ 'read_buffer': 0,
   \})
-
+  call ale#linter#PreventLoading('foobar2')
   call ale#linter#Define('foobar2', {
   \ 'name': 'buffer_linter',
   \ 'callback': 'TestCallback',
@@ -28,16 +44,10 @@ Before:
 
 After:
   Restore
-
-  noautocmd let &filetype = b:old_filetype
-  unlet b:old_filetype
   delfunction TestCallback
 
-  if has_key(g:ale_buffer_info, bufnr(''))
-    call remove(g:ale_buffer_info, bufnr(''))
-  endif
-
-  call ale#Queue(0)
+  call ale#linter#Reset()
+  call setloclist(0, [])
 
 Execute(Error should be removed when the filetype changes to something else we cannot check):
   call ale#Queue(0)