Async querying of CompilationDatabase supported

cpp/ycm/.ycm_clang_options.py

@@ -20,6 +20,7 @@ flags = [
 '-Wno-variadic-macros',
 '-fexceptions',
 '-DNDEBUG',
+'-DUSE_CLANG_COMPLETER',
 # THIS IS IMPORTANT! Without a "-std=<something>" flag, clang won't know which
 # language to use when compiling headers. So it will guess. Badly. So C++
 # headers will be compiled as C headers. You don't want that so ALWAYS specify
@@ -100,10 +101,11 @@ def FlagsForFile( filename ):
         filename )
     # Bear in mind that database.FlagsForFile does NOT return a python list, but
     # a "list-like" StringVec object
-    raw_flags = database.FlagsForFile( filename )
+    compilation_info = database.GetCompilationInfoForFile( filename )
     final_flags = PrepareClangFlags(
-        MakeRelativePathsInFlagsAbsolute( raw_flags,
+        MakeRelativePathsInFlagsAbsolute(
-                                          working_directory ),
+            compilation_info.compiler_flags_,
+            compilation_info.compiler_working_dir_ ),
         filename )
 
     # NOTE: This is just for YouCompleteMe; it's highly likely that your project

cpp/ycm/ClangCompleter/CompilationDatabase.cpp

@@ -20,19 +20,41 @@
 #include "standard.h"
 
 #include <boost/shared_ptr.hpp>
+#include <boost/bind.hpp>
+#include <boost/make_shared.hpp>
 #include <boost/type_traits/remove_pointer.hpp>
 
-using boost::shared_ptr;
+using boost::bind;
-using boost::shared_ptr;
+using boost::make_shared;
+using boost::packaged_task;
 using boost::remove_pointer;
+using boost::shared_ptr;
+using boost::thread;
+using boost::unique_future;
+using boost::function;
 
 namespace YouCompleteMe {
+
 typedef shared_ptr <
 remove_pointer< CXCompileCommands >::type > CompileCommandsWrap;
 
+
+void QueryThreadMain( CompilationDatabase::InfoTaskStack &info_task_stack ) {
+  while ( true ) {
+    try {
+      ( *info_task_stack.Pop() )();
+    } catch ( boost::thread_interrupted & ) {
+      return;
+    }
+  }
+
+}
+
+
 CompilationDatabase::CompilationDatabase(
   const std::string &path_to_directory )
-  : is_loaded_( false ) {
+  : threading_enabled_( false ),
+    is_loaded_( false ) {
   CXCompilationDatabase_Error status;
   compilation_database_ = clang_CompilationDatabase_fromDirectory(
                             path_to_directory.c_str(),
@@ -46,17 +68,27 @@ CompilationDatabase::~CompilationDatabase() {
 }
 
 
+// We need this mostly so that we can not use it in tests. Apparently the
+// GoogleTest framework goes apeshit on us if we enable threads by default.
+void CompilationDatabase::EnableThreading() {
+  threading_enabled_ = true;
+  InitThreads();
+}
+
+
 bool CompilationDatabase::DatabaseSuccessfullyLoaded() {
   return is_loaded_;
 }
 
 
-std::vector< std::string > CompilationDatabase::FlagsForFile(
+CompilationInfoForFile CompilationDatabase::GetCompilationInfoForFile(
-  const std::string &path_to_file ) {
+    const std::string &path_to_file ) {
-  std::vector< std::string > flags;
+  CompilationInfoForFile info;
 
   if ( !is_loaded_ )
-    return flags;
+    return info;
+
+  // TODO: mutex protect calls to getCompileCommands and getDirectory
 
   CompileCommandsWrap commands(
     clang_CompilationDatabase_getCompileCommands(
@@ -66,7 +98,7 @@ std::vector< std::string > CompilationDatabase::FlagsForFile(
   uint num_commands = clang_CompileCommands_getSize( commands.get() );
 
   if ( num_commands < 1 ) {
-    return flags;
+    return info;
   }
 
   // We always pick the first command offered
@@ -74,45 +106,47 @@ std::vector< std::string > CompilationDatabase::FlagsForFile(
                                commands.get(),
                                0 );
 
+  info.compiler_working_dir_ = CXStringToString(
+      clang_CompileCommand_getDirectory( command ) );
+
   uint num_flags = clang_CompileCommand_getNumArgs( command );
-  flags.reserve( num_flags );
+  info.compiler_flags_.reserve( num_flags );
 
   for ( uint i = 0; i < num_flags; ++i ) {
-    flags.push_back( CXStringToString(
+    info.compiler_flags_.push_back(
-                       clang_CompileCommand_getArg( command, i ) ) );
+        CXStringToString( clang_CompileCommand_getArg( command, i ) ) );
   }
 
-  return flags;
+  return info;
 }
 
 
-std::string CompilationDatabase::CompileCommandWorkingDirectoryForFile(
+Future< AsyncCompilationInfoForFile >
-  const std::string &path_to_file ) {
+CompilationDatabase::GetCompilationInfoForFileAsync(
-  std::string path_to_directory;
+    const std::string &path_to_file ) {
+  // TODO: throw exception when threading is not enabled and this is called
+  if ( !threading_enabled_ )
+    return Future< AsyncCompilationInfoForFile >();
 
-  if ( !is_loaded_ )
+  function< CompilationInfoForFile() > functor =
-    return path_to_directory;
+    bind( &CompilationDatabase::GetCompilationInfoForFile,
+          boost::ref( *this ),
+          path_to_file );
 
-  CompileCommandsWrap commands(
+  InfoTask task =
-    clang_CompilationDatabase_getCompileCommands(
+      make_shared< packaged_task< AsyncCompilationInfoForFile > >(
-      compilation_database_,
+          bind( ReturnValueAsShared< CompilationInfoForFile >,
-      path_to_file.c_str() ), clang_CompileCommands_dispose );
+                functor ) );
 
-  uint num_commands = clang_CompileCommands_getSize( commands.get() );
+  unique_future< AsyncCompilationInfoForFile > future = task->get_future();
+  info_task_stack_.Push( task );
+  return Future< AsyncCompilationInfoForFile >( boost::move( future ) );
+}
 
-  if ( num_commands < 1 ) {
-    return path_to_directory;
-  }
 
-  // We always pick the first command offered
+void CompilationDatabase::InitThreads() {
-  CXCompileCommand command = clang_CompileCommands_getCommand(
+  info_thread_ = boost::thread( QueryThreadMain,
-                               commands.get(),
+                                boost::ref( info_task_stack_ ) );
-                               0 );
-
-  path_to_directory = CXStringToString( clang_CompileCommand_getDirectory(
-                                          command ) );
-
-  return path_to_directory;
 }
 
 } // namespace YouCompleteMe

cpp/ycm/ClangCompleter/CompilationDatabase.h

@@ -18,13 +18,25 @@
 #ifndef COMPILATIONDATABASE_H_ZT7MQXPG
 #define COMPILATIONDATABASE_H_ZT7MQXPG
 
+#include "Future.h"
+#include "ConcurrentStack.h"
+
 #include <vector>
 #include <string>
 #include <boost/utility.hpp>
+#include <boost/shared_ptr.hpp>
 #include <clang-c/CXCompilationDatabase.h>
 
 namespace YouCompleteMe {
 
+struct CompilationInfoForFile {
+  std::vector< std::string > compiler_flags_;
+  std::string compiler_working_dir_;
+};
+
+typedef boost::shared_ptr< CompilationInfoForFile >
+AsyncCompilationInfoForFile;
+
 class CompilationDatabase : boost::noncopyable {
 public:
   CompilationDatabase( const std::string &path_to_directory );
@@ -32,14 +44,28 @@ public:
 
   bool DatabaseSuccessfullyLoaded();
 
-  std::vector< std::string > FlagsForFile( const std::string &path_to_file );
+  void EnableThreading();
 
-  std::string CompileCommandWorkingDirectoryForFile(
+  CompilationInfoForFile GetCompilationInfoForFile(
-    const std::string &path_to_file );
+      const std::string &path_to_file );
+
+  Future< AsyncCompilationInfoForFile > GetCompilationInfoForFileAsync(
+      const std::string &path_to_file );
+
+  typedef boost::shared_ptr <
+  boost::packaged_task< AsyncCompilationInfoForFile > > InfoTask;
+
+  typedef ConcurrentStack< InfoTask > InfoTaskStack;
 
 private:
+  void InitThreads();
+
+  bool threading_enabled_;
   bool is_loaded_;
   CXCompilationDatabase compilation_database_;
+
+  boost::thread info_thread_;
+  InfoTaskStack info_task_stack_;
 };
 
 } // namespace YouCompleteMe

cpp/ycm/ycm_core.cpp

@@ -74,6 +74,13 @@ BOOST_PYTHON_MODULE(ycm_core)
     .def( "ResultsReady", &Future< AsyncCompletions >::ResultsReady )
     .def( "GetResults", &Future< AsyncCompletions >::GetResults );
 
+  class_< Future< AsyncCompilationInfoForFile > >(
+      "FutureCompilationInfoForFile" )
+    .def( "ResultsReady",
+          &Future< AsyncCompilationInfoForFile >::ResultsReady )
+    .def( "GetResults",
+          &Future< AsyncCompilationInfoForFile >::GetResults );
+
   // CAREFUL HERE! For filename and contents we are referring directly to
   // Python-allocated and -managed memory since we are accepting pointers to
   // data members of python objects. We need to ensure that those objects
@@ -127,11 +134,24 @@ BOOST_PYTHON_MODULE(ycm_core)
 
   class_< CompilationDatabase, boost::noncopyable >(
       "CompilationDatabase", init< std::string >() )
+    .def( "EnableThreading", &CompilationDatabase::EnableThreading )
     .def( "FlagsForFile", &CompilationDatabase::FlagsForFile )
     .def( "DatabaseSuccessfullyLoaded",
           &CompilationDatabase::DatabaseSuccessfullyLoaded )
     .def( "CompileCommandWorkingDirectoryForFile",
-          &CompilationDatabase::CompileCommandWorkingDirectoryForFile );
+          &CompilationDatabase::CompileCommandWorkingDirectoryForFile )
+    .def( "GetCompilationInfoForFile",
+          &CompilationDatabase::GetCompilationInfoForFile )
+    .def( "GetCompilationInfoForFileAsync",
+          &CompilationDatabase::GetCompilationInfoForFileAsync );
+
+  class_< CompilationInfoForFile,
+      boost::shared_ptr< CompilationInfoForFile > >(
+          "CompilationInfoForFile", no_init )
+    .def_readonly( "compiler_working_dir_",
+                   &CompilationInfoForFile::compiler_working_dir_ )
+    .def_readonly( "compiler_flags_",
+                   &CompilationInfoForFile::compiler_flags_ );
 
 #endif // USE_CLANG_COMPLETER
 }

python/completers/all/identifier_completer.py

@@ -46,7 +46,7 @@ class IdentifierCompleter( Completer ):
 
   def CandidatesForQueryAsync( self, query ):
     filetype = vim.eval( "&filetype" )
-    self.future = self.completer.CandidatesForQueryAndTypeAsync(
+    self.completions_future = self.completer.CandidatesForQueryAndTypeAsync(
       utils.SanitizeQuery( query ),
       filetype )
 
@@ -109,9 +109,9 @@ class IdentifierCompleter( Completer ):
 
 
   def CandidatesFromStoredRequest( self ):
-    if not self.future:
+    if not self.completions_future:
       return []
-    completions = self.future.GetResults()[
+    completions = self.completions_future.GetResults()[
       : MAX_IDENTIFIER_COMPLETIONS_RETURNED ]
 
     # We will never have duplicates in completions so with 'dup':1 we tell Vim

python/completers/completer.py

@@ -25,21 +25,21 @@ class Completer( object ):
 
 
   def __init__( self ):
-    self.future = None
+    self.completions_future = None
 
 
   def AsyncCandidateRequestReady( self ):
-    if not self.future:
+    if not self.completions_future:
       # We return True so that the caller can extract the default value from the
       # future
       return True
-    return self.future.ResultsReady()
+    return self.completions_future.ResultsReady()
 
 
   def CandidatesFromStoredRequest( self ):
-    if not self.future:
+    if not self.completions_future:
       return []
-    return self.future.GetResults()
+    return self.completions_future.GetResults()
 
 
   def OnFileReadyToParse( self ):

python/completers/cpp/clang_completer.py

@@ -79,7 +79,14 @@ class ClangCompleter( Completer ):
   def CandidatesForQueryAsync( self, query ):
     if self.completer.UpdatingTranslationUnit( vim.current.buffer.name ):
       vimsupport.PostVimMessage( 'Still parsing file, no completions yet.' )
-      self.future = None
+      self.completions_future = None
+      return
+
+    filename = vim.current.buffer.name
+    flags = self.flags.FlagsForFile( filename )
+    if not flags:
+      vimsupport.PostVimMessage( 'Still no compile flags, no completions yet.' )
+      self.completions_future = None
       return
 
     # TODO: sanitize query, probably in C++ code
@@ -90,21 +97,21 @@ class ClangCompleter( Completer ):
 
     line, _ = vim.current.window.cursor
     column = int( vim.eval( "s:completion_start_column" ) ) + 1
-    current_buffer = vim.current.buffer
+    self.completions_future = (
-    # TODO: rename future to completions_future
+      self.completer.CandidatesForQueryAndLocationInFileAsync(
-    self.future = self.completer.CandidatesForQueryAndLocationInFileAsync(
+        query,
-      query,
+        filename,
-      current_buffer.name,
+        line,
-      line,
+        column,
-      column,
+        files,
-      files,
+        flags ) )
-      self.flags.FlagsForFile( current_buffer.name ) )
 
 
   def CandidatesFromStoredRequest( self ):
-    if not self.future:
+    if not self.completions_future:
       return []
-    results = [ CompletionDataToDict( x ) for x in self.future.GetResults() ]
+    results = [ CompletionDataToDict( x ) for x in
+                self.completions_future.GetResults() ]
     if not results:
       vimsupport.PostVimMessage( 'No completions found; errors in the file?' )
     return results
@@ -112,13 +119,19 @@ class ClangCompleter( Completer ):
 
   def OnFileReadyToParse( self ):
     if vimsupport.NumLinesInBuffer( vim.current.buffer ) < 5:
+      self.parse_future = None
       return
 
     filename = vim.current.buffer.name
+    flags = self.flags.FlagsForFile( filename )
+    if not flags:
+      self.parse_future = None
+      return
+
     self.parse_future = self.completer.UpdateTranslationUnitAsync(
       filename,
       self.GetUnsavedFilesVector(),
-      self.flags.FlagsForFile( filename ) )
+      flags )
 
 
   def DiagnosticsForCurrentFileReady( self ):

python/completers/cpp/flags.py

@@ -46,9 +46,13 @@ class Flags( object ):
       flags_module = self._FlagsModuleForFile( filename )
       if not flags_module:
         vimsupport.PostVimMessage( NO_OPTIONS_FILENAME_MESSAGE )
-        return ycm_core.StringVec()
+        return None
 
       results = flags_module.FlagsForFile( filename )
+
+      if not results.get( 'flags_ready', True ):
+        return None
+
       results[ 'flags' ] += self.special_clang_flags
       sanitized_flags = _SanitizeFlags( results[ 'flags' ] )