Async clang parsing of the current file

autoload/youcompleteme.vim

@@ -83,20 +83,23 @@ endfunction
 
 function! s:OnBufferVisit()
   call s:SetCompleteFunc()
-  py identcomp.OnFileEnter()
+  call s:ParseFile()
-
-  if g:ycm_clang_completion_enabled && pyeval('ycm.ClangAvailableForFile()')
-    py clangcomp.OnFileEnter()
-  endif
 endfunction
 
 
 function! s:OnCursorHold()
-  " TODO: make this async, it's causing lag
+  call s:ParseFile()
-  py identcomp.AddBufferIdentifiers()
 endfunction
 
 
+function! s:ParseFile()
+  py identcomp.OnFileReadyToParse()
+
+  if g:ycm_clang_completion_enabled && pyeval('ycm.ClangAvailableForFile()')
+    py clangcomp.OnFileReadyToParse()
+  endif
+endfunction
+
 function! s:SetCompleteFunc()
   let &completefunc = 'youcompleteme#Complete'
   let &l:completefunc = 'youcompleteme#Complete'

cpp/ycm/ClangCompleter.cpp

@@ -27,6 +27,8 @@
 #include <clang-c/Index.h>
 #include <boost/make_shared.hpp>
 
+// TODO: remove all explicit uses of the boost:: prefix by adding explicit using
+// directives for the stuff we need
 namespace fs = boost::filesystem;
 using boost::packaged_task;
 using boost::bind;
@@ -35,6 +37,8 @@ using boost::make_shared;
 using boost::shared_ptr;
 using boost::bind;
 using boost::thread;
+using boost::lock_guard;
+using boost::mutex;
 
 namespace YouCompleteMe
 {
@@ -266,6 +270,13 @@ void ClangCompleter::SetFileCompileFlags(
 }
 
 
+bool ClangCompleter::UpdatingTranslationUnit()
+{
+  lock_guard< mutex > lock( file_parse_task_mutex_ );
+  return bool( file_parse_task_ );
+}
+
+
 void ClangCompleter::UpdateTranslationUnit(
     const std::string &filename,
     const std::vector< UnsavedFile > &unsaved_files )
@@ -293,6 +304,27 @@ void ClangCompleter::UpdateTranslationUnit(
 }
 
 
+void ClangCompleter::UpdateTranslationUnitAsync(
+    std::string filename,
+    std::vector< UnsavedFile > unsaved_files )
+{
+  boost::function< void() > functor =
+    bind( &ClangCompleter::UpdateTranslationUnit,
+          boost::ref( *this ),
+          boost::move( filename ),
+          boost::move( unsaved_files ) );
+
+  boost::lock_guard< boost::mutex > lock( file_parse_task_mutex_ );
+
+  // Only ever set the task when it's NULL; if it's not, that means that the
+  // clang thread is working on it
+  if ( !file_parse_task_ ) {
+    file_parse_task_ = make_shared< packaged_task< void > >( functor );
+    file_parse_task_condition_variable_.notify_all();
+  }
+}
+
+
 std::vector< CompletionData > ClangCompleter::CandidatesForLocationInFile(
     const std::string &filename,
     int line,
@@ -341,6 +373,10 @@ ClangCompleter::CandidatesForQueryAndLocationInFileAsync(
 
   if ( query.empty() )
   {
+    // The clang thread is busy, return nothing
+    if ( UpdatingTranslationUnit() )
+      return Future< AsyncCompletions >();
+
     {
       boost::lock_guard< boost::mutex > lock( clang_data_ready_mutex_ );
       clang_data_ready_ = false;
@@ -386,7 +422,7 @@ ClangCompleter::CandidatesForQueryAndLocationInFileAsync(
         bind( ReturnValueAsShared< std::vector< CompletionData > >,
               candidates_for_location_in_file_functor ) );
 
-    clang_task_.Set( task );
+    clang_completions_task_.Set( task );
   }
 
   return Future< AsyncCompletions >( boost::move( future ) );
@@ -408,7 +444,7 @@ CXTranslationUnit ClangCompleter::CreateTranslationUnit(
   std::vector< CXUnsavedFile > cxunsaved_files = ToCXUnsavedFiles(
       unsaved_files );
 
-  return clang_parseTranslationUnit(
+  CXTranslationUnit unit = clang_parseTranslationUnit(
       clang_index_,
       filename.c_str(),
       &flags[ 0 ],
@@ -416,6 +452,16 @@ CXTranslationUnit ClangCompleter::CreateTranslationUnit(
       &cxunsaved_files[ 0 ],
       cxunsaved_files.size(),
       clang_defaultEditingTranslationUnitOptions() );
+
+  // Only with a reparse is the preable precompiled. I do not know why...
+  // TODO: report this bug on the clang tracker
+  clang_reparseTranslationUnit(
+      unit,
+      cxunsaved_files.size(),
+      &cxunsaved_files[ 0 ],
+      clang_defaultEditingTranslationUnitOptions() );
+
+  return unit;
 }
 
 
@@ -516,21 +562,56 @@ void ClangCompleter::InitThreads()
   {
     sorting_threads_.create_thread(
         bind( &ClangCompleter::SortingThreadMain,
-              boost::ref( *this ),
+              boost::ref( *this ) ) );
-              boost::ref( sorting_task_ ) ) );
   }
 
-  clang_thread_ = boost::thread( &ClangCompleter::ClangThreadMain,
+  clang_completions_thread_ = boost::thread(
-                                 boost::ref( *this ),
+      &ClangCompleter::ClangCompletionsThreadMain,
-                                 boost::ref( clang_task_ ) );
+      boost::ref( *this ) );
+
+  file_parse_thread_ = boost::thread(
+      &ClangCompleter::FileParseThreadMain,
+      boost::ref( *this ) );
 }
 
 
-void ClangCompleter::ClangThreadMain( LatestTask &clang_task )
+void ClangCompleter::FileParseThreadMain()
 {
   while ( true )
   {
-    shared_ptr< packaged_task< AsyncCompletions > > task = clang_task.Get();
+    {
+      boost::unique_lock< boost::mutex > lock( file_parse_task_mutex_ );
+
+      while ( !file_parse_task_ )
+      {
+        file_parse_task_condition_variable_.wait( lock );
+      }
+    }
+
+    ( *file_parse_task_ )();
+
+    lock_guard< mutex > lock( file_parse_task_mutex_ );
+    file_parse_task_ = VoidTask();
+  }
+}
+
+
+void ClangCompleter::ClangCompletionsThreadMain()
+{
+  while ( true )
+  {
+    // TODO: this should be a separate func, much like the file_parse_task_ part
+    shared_ptr< packaged_task< AsyncCompletions > > task =
+      clang_completions_task_.Get();
+
+    // If the file parse thread is accessing clang by parsing a file, then drop
+    // the current completion request
+    {
+      lock_guard< mutex > lock( file_parse_task_mutex_ );
+      if ( file_parse_task_ )
+        continue;
+    }
+
     ( *task )();
     unique_future< AsyncCompletions > future = task->get_future();
 
@@ -550,7 +631,7 @@ void ClangCompleter::ClangThreadMain( LatestTask &clang_task )
 }
 
 
-void ClangCompleter::SortingThreadMain( LatestTask &sorting_task )
+void ClangCompleter::SortingThreadMain()
 {
   while ( true )
   {
@@ -565,7 +646,8 @@ void ClangCompleter::SortingThreadMain( LatestTask &sorting_task )
         }
       }
 
-      shared_ptr< packaged_task< AsyncCompletions > > task = sorting_task.Get();
+      shared_ptr< packaged_task< AsyncCompletions > > task =
+        sorting_task_.Get();
 
       {
         boost::shared_lock< boost::shared_mutex > reader_lock(

cpp/ycm/ClangCompleter.h

@@ -60,9 +60,15 @@ public:
   void SetFileCompileFlags( const std::string &filename,
                             const std::vector< std::string > &flags );
 
+  bool UpdatingTranslationUnit();
+
   void UpdateTranslationUnit( const std::string &filename,
                               const std::vector< UnsavedFile > &unsaved_files );
 
+  void UpdateTranslationUnitAsync(
+      std::string filename,
+      std::vector< UnsavedFile > unsaved_files );
+
   std::vector< CompletionData > CandidatesForLocationInFile(
       const std::string &filename,
       int line,
@@ -98,9 +104,11 @@ private:
 
   void InitThreads();
 
-  void ClangThreadMain( LatestTask &clang_task );
+  void FileParseThreadMain();
 
-  void SortingThreadMain( LatestTask &sorting_task );
+  void ClangCompletionsThreadMain();
+
+  void SortingThreadMain();
 
 
   /////////////////////////////
@@ -117,10 +125,19 @@ private:
 
   CandidateRepository &candidate_repository_;
 
-  mutable LatestTask clang_task_;
+  // TODO: move everything thread-related into a separated helper class
+  mutable LatestTask clang_completions_task_;
 
   mutable LatestTask sorting_task_;
 
+  // Only the clang thread can make this NULL and only the GUI thread can make
+  // it non-NULL. The file_parse_task_mutex_ is used before checking the state
+  // of NULL. [ NULL for a shared_ptr naturally means default-constructed
+  // shared_ptr]
+  VoidTask file_parse_task_;
+  boost::mutex file_parse_task_mutex_;
+  boost::condition_variable file_parse_task_condition_variable_;
+
   bool threading_enabled_;
 
   // TODO: use boost.atomic for clang_data_ready_
@@ -131,10 +148,12 @@ private:
   std::vector< CompletionData > latest_clang_results_;
   boost::shared_mutex latest_clang_results_shared_mutex_;
 
-  // Unfortunately clang is not thread-safe so we can only ever use one thread
+  // Unfortunately clang is not thread-safe so we need to be careful when we
-  // to access it. So this one background thread will be the only thread that
+  // access it. Only one thread at a time is allowed to access any single
-  // can access libclang.
+  // translation unit.
-  boost::thread clang_thread_;
+  boost::thread clang_completions_thread_;
+
+  boost::thread file_parse_thread_;
 
   boost::thread_group sorting_threads_;
 };

cpp/ycm/Future.h

@@ -27,7 +27,7 @@ namespace YouCompleteMe
 {
 
 class Result;
-template< typename T > class ConcurrentLatestValue;
+typedef boost::shared_ptr< boost::packaged_task< void > > VoidTask;
 
 template< typename T >
 boost::shared_ptr< T > ReturnValueAsShared(
@@ -47,6 +47,13 @@ public:
 
   bool ResultsReady()
   {
+    // It's OK to return true since GetResults will just return a
+    // default-constructed value if the future_ is uninitialized. If we don't
+    // return true for this case, any loop waiting on ResultsReady will wait
+    // forever.
+    if ( future_.get_state() == boost::future_state::uninitialized )
+      return true;
+
     return future_.is_ready();
   }
 

cpp/ycm/IdentifierCompleter.h

@@ -58,8 +58,6 @@ typedef boost::shared_ptr<
 
 typedef ConcurrentLatestValue< QueryTask > LatestQueryTask;
 
-typedef boost::shared_ptr< boost::packaged_task< void > > VoidTask;
-
 typedef ConcurrentStack< VoidTask > BufferIdentifiersTaskStack;
 
 

cpp/ycm/indexer.cpp

@@ -86,7 +86,9 @@ BOOST_PYTHON_MODULE(indexer)
     .def( "EnableThreading", &ClangCompleter::EnableThreading )
     .def( "SetGlobalCompileFlags", &ClangCompleter::SetGlobalCompileFlags )
     .def( "SetFileCompileFlags", &ClangCompleter::SetFileCompileFlags )
-    .def( "UpdateTranslationUnit", &ClangCompleter::UpdateTranslationUnit )
+    .def( "UpdatingTranslationUnit", &ClangCompleter::UpdatingTranslationUnit )
+    .def( "UpdateTranslationUnitAsync",
+          &ClangCompleter::UpdateTranslationUnitAsync )
     .def( "CandidatesForQueryAndLocationInFileAsync",
           &ClangCompleter::CandidatesForQueryAndLocationInFileAsync );
 }

python/ycm.py

@@ -33,6 +33,10 @@ class Completer( object ):
 
 
   def AsyncCandidateRequestReady( self ):
+    if not self.future:
+      # We return True so that the caller can extract the default value from the
+      # future
+      return True
     return self.future.ResultsReady()
 
 
@@ -42,7 +46,7 @@ class Completer( object ):
     return self.future.GetResults()
 
 
-  def OnFileEnter( self ):
+  def OnFileReadyToParse( self ):
     pass
 
 
@@ -90,7 +94,7 @@ class IdentifierCompleter( Completer ):
                                                            filepath )
 
 
-  def OnFileEnter( self ):
+  def OnFileReadyToParse( self ):
     self.AddBufferIdentifiers()
 
 
@@ -134,6 +138,11 @@ class ClangCompleter( Completer ):
 
 
   def CandidatesForQueryAsync( self, query ):
+    if self.completer.UpdatingTranslationUnit():
+      PostVimMessage( 'Still parsing file, no completions yet.' )
+      self.future = None
+      return
+
     # TODO: sanitize query
 
     # CAREFUL HERE! For UnsavedFile filename and contents we are referring
@@ -165,12 +174,17 @@ class ClangCompleter( Completer ):
     return [ CompletionDataToDict( x ) for x in self.future.GetResults() ]
 
 
-  def OnFileEnter( self ):
+  def OnFileReadyToParse( self ):
-    self.future = self.completer.UpdateTranslationUnit(
+    self.future = self.completer.UpdateTranslationUnitAsync(
       vim.current.buffer.name,
       self.GetUnsavedFilesVector() )
 
 
+def PostVimMessage( message ):
+  # TODO: escape the message string before formating it
+  vim.command( 'echohl WarningMsg | echomsg "{0}" | echohl None'
+               .format( message ) )
+
 
 def GetUnsavedBuffers():
   def BufferModified( buffer_number ):