Async querying of CompilationDatabase supported

This commit is contained in:
Strahinja Val Markovic 2013-01-23 17:23:51 -08:00
parent 0b60b8d03b
commit 90fe31f98e
8 changed files with 162 additions and 63 deletions

View File

@ -20,6 +20,7 @@ flags = [
'-Wno-variadic-macros', '-Wno-variadic-macros',
'-fexceptions', '-fexceptions',
'-DNDEBUG', '-DNDEBUG',
'-DUSE_CLANG_COMPLETER',
# THIS IS IMPORTANT! Without a "-std=<something>" flag, clang won't know which # THIS IS IMPORTANT! Without a "-std=<something>" flag, clang won't know which
# language to use when compiling headers. So it will guess. Badly. So C++ # language to use when compiling headers. So it will guess. Badly. So C++
# headers will be compiled as C headers. You don't want that so ALWAYS specify # headers will be compiled as C headers. You don't want that so ALWAYS specify
@ -100,10 +101,11 @@ def FlagsForFile( filename ):
filename ) filename )
# Bear in mind that database.FlagsForFile does NOT return a python list, but # Bear in mind that database.FlagsForFile does NOT return a python list, but
# a "list-like" StringVec object # a "list-like" StringVec object
raw_flags = database.FlagsForFile( filename ) compilation_info = database.GetCompilationInfoForFile( filename )
final_flags = PrepareClangFlags( final_flags = PrepareClangFlags(
MakeRelativePathsInFlagsAbsolute( raw_flags, MakeRelativePathsInFlagsAbsolute(
working_directory ), compilation_info.compiler_flags_,
compilation_info.compiler_working_dir_ ),
filename ) filename )
# NOTE: This is just for YouCompleteMe; it's highly likely that your project # NOTE: This is just for YouCompleteMe; it's highly likely that your project

View File

@ -20,19 +20,41 @@
#include "standard.h" #include "standard.h"
#include <boost/shared_ptr.hpp> #include <boost/shared_ptr.hpp>
#include <boost/bind.hpp>
#include <boost/make_shared.hpp>
#include <boost/type_traits/remove_pointer.hpp> #include <boost/type_traits/remove_pointer.hpp>
using boost::shared_ptr; using boost::bind;
using boost::shared_ptr; using boost::make_shared;
using boost::packaged_task;
using boost::remove_pointer; using boost::remove_pointer;
using boost::shared_ptr;
using boost::thread;
using boost::unique_future;
using boost::function;
namespace YouCompleteMe { namespace YouCompleteMe {
typedef shared_ptr < typedef shared_ptr <
remove_pointer< CXCompileCommands >::type > CompileCommandsWrap; remove_pointer< CXCompileCommands >::type > CompileCommandsWrap;
void QueryThreadMain( CompilationDatabase::InfoTaskStack &info_task_stack ) {
while ( true ) {
try {
( *info_task_stack.Pop() )();
} catch ( boost::thread_interrupted & ) {
return;
}
}
}
CompilationDatabase::CompilationDatabase( CompilationDatabase::CompilationDatabase(
const std::string &path_to_directory ) const std::string &path_to_directory )
: is_loaded_( false ) { : threading_enabled_( false ),
is_loaded_( false ) {
CXCompilationDatabase_Error status; CXCompilationDatabase_Error status;
compilation_database_ = clang_CompilationDatabase_fromDirectory( compilation_database_ = clang_CompilationDatabase_fromDirectory(
path_to_directory.c_str(), path_to_directory.c_str(),
@ -46,17 +68,27 @@ CompilationDatabase::~CompilationDatabase() {
} }
// We need this mostly so that we can not use it in tests. Apparently the
// GoogleTest framework goes apeshit on us if we enable threads by default.
void CompilationDatabase::EnableThreading() {
threading_enabled_ = true;
InitThreads();
}
bool CompilationDatabase::DatabaseSuccessfullyLoaded() { bool CompilationDatabase::DatabaseSuccessfullyLoaded() {
return is_loaded_; return is_loaded_;
} }
std::vector< std::string > CompilationDatabase::FlagsForFile( CompilationInfoForFile CompilationDatabase::GetCompilationInfoForFile(
const std::string &path_to_file ) { const std::string &path_to_file ) {
std::vector< std::string > flags; CompilationInfoForFile info;
if ( !is_loaded_ ) if ( !is_loaded_ )
return flags; return info;
// TODO: mutex protect calls to getCompileCommands and getDirectory
CompileCommandsWrap commands( CompileCommandsWrap commands(
clang_CompilationDatabase_getCompileCommands( clang_CompilationDatabase_getCompileCommands(
@ -66,7 +98,7 @@ std::vector< std::string > CompilationDatabase::FlagsForFile(
uint num_commands = clang_CompileCommands_getSize( commands.get() ); uint num_commands = clang_CompileCommands_getSize( commands.get() );
if ( num_commands < 1 ) { if ( num_commands < 1 ) {
return flags; return info;
} }
// We always pick the first command offered // We always pick the first command offered
@ -74,45 +106,47 @@ std::vector< std::string > CompilationDatabase::FlagsForFile(
commands.get(), commands.get(),
0 ); 0 );
info.compiler_working_dir_ = CXStringToString(
clang_CompileCommand_getDirectory( command ) );
uint num_flags = clang_CompileCommand_getNumArgs( command ); uint num_flags = clang_CompileCommand_getNumArgs( command );
flags.reserve( num_flags ); info.compiler_flags_.reserve( num_flags );
for ( uint i = 0; i < num_flags; ++i ) { for ( uint i = 0; i < num_flags; ++i ) {
flags.push_back( CXStringToString( info.compiler_flags_.push_back(
clang_CompileCommand_getArg( command, i ) ) ); CXStringToString( clang_CompileCommand_getArg( command, i ) ) );
} }
return flags; return info;
} }
std::string CompilationDatabase::CompileCommandWorkingDirectoryForFile( Future< AsyncCompilationInfoForFile >
CompilationDatabase::GetCompilationInfoForFileAsync(
const std::string &path_to_file ) { const std::string &path_to_file ) {
std::string path_to_directory; // TODO: throw exception when threading is not enabled and this is called
if ( !threading_enabled_ )
return Future< AsyncCompilationInfoForFile >();
if ( !is_loaded_ ) function< CompilationInfoForFile() > functor =
return path_to_directory; bind( &CompilationDatabase::GetCompilationInfoForFile,
boost::ref( *this ),
path_to_file );
CompileCommandsWrap commands( InfoTask task =
clang_CompilationDatabase_getCompileCommands( make_shared< packaged_task< AsyncCompilationInfoForFile > >(
compilation_database_, bind( ReturnValueAsShared< CompilationInfoForFile >,
path_to_file.c_str() ), clang_CompileCommands_dispose ); functor ) );
uint num_commands = clang_CompileCommands_getSize( commands.get() ); unique_future< AsyncCompilationInfoForFile > future = task->get_future();
info_task_stack_.Push( task );
return Future< AsyncCompilationInfoForFile >( boost::move( future ) );
}
if ( num_commands < 1 ) {
return path_to_directory;
}
// We always pick the first command offered void CompilationDatabase::InitThreads() {
CXCompileCommand command = clang_CompileCommands_getCommand( info_thread_ = boost::thread( QueryThreadMain,
commands.get(), boost::ref( info_task_stack_ ) );
0 );
path_to_directory = CXStringToString( clang_CompileCommand_getDirectory(
command ) );
return path_to_directory;
} }
} // namespace YouCompleteMe } // namespace YouCompleteMe

View File

@ -18,13 +18,25 @@
#ifndef COMPILATIONDATABASE_H_ZT7MQXPG #ifndef COMPILATIONDATABASE_H_ZT7MQXPG
#define COMPILATIONDATABASE_H_ZT7MQXPG #define COMPILATIONDATABASE_H_ZT7MQXPG
#include "Future.h"
#include "ConcurrentStack.h"
#include <vector> #include <vector>
#include <string> #include <string>
#include <boost/utility.hpp> #include <boost/utility.hpp>
#include <boost/shared_ptr.hpp>
#include <clang-c/CXCompilationDatabase.h> #include <clang-c/CXCompilationDatabase.h>
namespace YouCompleteMe { namespace YouCompleteMe {
struct CompilationInfoForFile {
std::vector< std::string > compiler_flags_;
std::string compiler_working_dir_;
};
typedef boost::shared_ptr< CompilationInfoForFile >
AsyncCompilationInfoForFile;
class CompilationDatabase : boost::noncopyable { class CompilationDatabase : boost::noncopyable {
public: public:
CompilationDatabase( const std::string &path_to_directory ); CompilationDatabase( const std::string &path_to_directory );
@ -32,14 +44,28 @@ public:
bool DatabaseSuccessfullyLoaded(); bool DatabaseSuccessfullyLoaded();
std::vector< std::string > FlagsForFile( const std::string &path_to_file ); void EnableThreading();
std::string CompileCommandWorkingDirectoryForFile( CompilationInfoForFile GetCompilationInfoForFile(
const std::string &path_to_file ); const std::string &path_to_file );
Future< AsyncCompilationInfoForFile > GetCompilationInfoForFileAsync(
const std::string &path_to_file );
typedef boost::shared_ptr <
boost::packaged_task< AsyncCompilationInfoForFile > > InfoTask;
typedef ConcurrentStack< InfoTask > InfoTaskStack;
private: private:
void InitThreads();
bool threading_enabled_;
bool is_loaded_; bool is_loaded_;
CXCompilationDatabase compilation_database_; CXCompilationDatabase compilation_database_;
boost::thread info_thread_;
InfoTaskStack info_task_stack_;
}; };
} // namespace YouCompleteMe } // namespace YouCompleteMe

View File

@ -74,6 +74,13 @@ BOOST_PYTHON_MODULE(ycm_core)
.def( "ResultsReady", &Future< AsyncCompletions >::ResultsReady ) .def( "ResultsReady", &Future< AsyncCompletions >::ResultsReady )
.def( "GetResults", &Future< AsyncCompletions >::GetResults ); .def( "GetResults", &Future< AsyncCompletions >::GetResults );
class_< Future< AsyncCompilationInfoForFile > >(
"FutureCompilationInfoForFile" )
.def( "ResultsReady",
&Future< AsyncCompilationInfoForFile >::ResultsReady )
.def( "GetResults",
&Future< AsyncCompilationInfoForFile >::GetResults );
// CAREFUL HERE! For filename and contents we are referring directly to // CAREFUL HERE! For filename and contents we are referring directly to
// Python-allocated and -managed memory since we are accepting pointers to // Python-allocated and -managed memory since we are accepting pointers to
// data members of python objects. We need to ensure that those objects // data members of python objects. We need to ensure that those objects
@ -127,11 +134,24 @@ BOOST_PYTHON_MODULE(ycm_core)
class_< CompilationDatabase, boost::noncopyable >( class_< CompilationDatabase, boost::noncopyable >(
"CompilationDatabase", init< std::string >() ) "CompilationDatabase", init< std::string >() )
.def( "EnableThreading", &CompilationDatabase::EnableThreading )
.def( "FlagsForFile", &CompilationDatabase::FlagsForFile ) .def( "FlagsForFile", &CompilationDatabase::FlagsForFile )
.def( "DatabaseSuccessfullyLoaded", .def( "DatabaseSuccessfullyLoaded",
&CompilationDatabase::DatabaseSuccessfullyLoaded ) &CompilationDatabase::DatabaseSuccessfullyLoaded )
.def( "CompileCommandWorkingDirectoryForFile", .def( "CompileCommandWorkingDirectoryForFile",
&CompilationDatabase::CompileCommandWorkingDirectoryForFile ); &CompilationDatabase::CompileCommandWorkingDirectoryForFile )
.def( "GetCompilationInfoForFile",
&CompilationDatabase::GetCompilationInfoForFile )
.def( "GetCompilationInfoForFileAsync",
&CompilationDatabase::GetCompilationInfoForFileAsync );
class_< CompilationInfoForFile,
boost::shared_ptr< CompilationInfoForFile > >(
"CompilationInfoForFile", no_init )
.def_readonly( "compiler_working_dir_",
&CompilationInfoForFile::compiler_working_dir_ )
.def_readonly( "compiler_flags_",
&CompilationInfoForFile::compiler_flags_ );
#endif // USE_CLANG_COMPLETER #endif // USE_CLANG_COMPLETER
} }

View File

@ -46,7 +46,7 @@ class IdentifierCompleter( Completer ):
def CandidatesForQueryAsync( self, query ): def CandidatesForQueryAsync( self, query ):
filetype = vim.eval( "&filetype" ) filetype = vim.eval( "&filetype" )
self.future = self.completer.CandidatesForQueryAndTypeAsync( self.completions_future = self.completer.CandidatesForQueryAndTypeAsync(
utils.SanitizeQuery( query ), utils.SanitizeQuery( query ),
filetype ) filetype )
@ -109,9 +109,9 @@ class IdentifierCompleter( Completer ):
def CandidatesFromStoredRequest( self ): def CandidatesFromStoredRequest( self ):
if not self.future: if not self.completions_future:
return [] return []
completions = self.future.GetResults()[ completions = self.completions_future.GetResults()[
: MAX_IDENTIFIER_COMPLETIONS_RETURNED ] : MAX_IDENTIFIER_COMPLETIONS_RETURNED ]
# We will never have duplicates in completions so with 'dup':1 we tell Vim # We will never have duplicates in completions so with 'dup':1 we tell Vim

View File

@ -25,21 +25,21 @@ class Completer( object ):
def __init__( self ): def __init__( self ):
self.future = None self.completions_future = None
def AsyncCandidateRequestReady( self ): def AsyncCandidateRequestReady( self ):
if not self.future: if not self.completions_future:
# We return True so that the caller can extract the default value from the # We return True so that the caller can extract the default value from the
# future # future
return True return True
return self.future.ResultsReady() return self.completions_future.ResultsReady()
def CandidatesFromStoredRequest( self ): def CandidatesFromStoredRequest( self ):
if not self.future: if not self.completions_future:
return [] return []
return self.future.GetResults() return self.completions_future.GetResults()
def OnFileReadyToParse( self ): def OnFileReadyToParse( self ):

View File

@ -79,7 +79,14 @@ class ClangCompleter( Completer ):
def CandidatesForQueryAsync( self, query ): def CandidatesForQueryAsync( self, query ):
if self.completer.UpdatingTranslationUnit( vim.current.buffer.name ): if self.completer.UpdatingTranslationUnit( vim.current.buffer.name ):
vimsupport.PostVimMessage( 'Still parsing file, no completions yet.' ) vimsupport.PostVimMessage( 'Still parsing file, no completions yet.' )
self.future = None self.completions_future = None
return
filename = vim.current.buffer.name
flags = self.flags.FlagsForFile( filename )
if not flags:
vimsupport.PostVimMessage( 'Still no compile flags, no completions yet.' )
self.completions_future = None
return return
# TODO: sanitize query, probably in C++ code # TODO: sanitize query, probably in C++ code
@ -90,21 +97,21 @@ class ClangCompleter( Completer ):
line, _ = vim.current.window.cursor line, _ = vim.current.window.cursor
column = int( vim.eval( "s:completion_start_column" ) ) + 1 column = int( vim.eval( "s:completion_start_column" ) ) + 1
current_buffer = vim.current.buffer self.completions_future = (
# TODO: rename future to completions_future self.completer.CandidatesForQueryAndLocationInFileAsync(
self.future = self.completer.CandidatesForQueryAndLocationInFileAsync(
query, query,
current_buffer.name, filename,
line, line,
column, column,
files, files,
self.flags.FlagsForFile( current_buffer.name ) ) flags ) )
def CandidatesFromStoredRequest( self ): def CandidatesFromStoredRequest( self ):
if not self.future: if not self.completions_future:
return [] return []
results = [ CompletionDataToDict( x ) for x in self.future.GetResults() ] results = [ CompletionDataToDict( x ) for x in
self.completions_future.GetResults() ]
if not results: if not results:
vimsupport.PostVimMessage( 'No completions found; errors in the file?' ) vimsupport.PostVimMessage( 'No completions found; errors in the file?' )
return results return results
@ -112,13 +119,19 @@ class ClangCompleter( Completer ):
def OnFileReadyToParse( self ): def OnFileReadyToParse( self ):
if vimsupport.NumLinesInBuffer( vim.current.buffer ) < 5: if vimsupport.NumLinesInBuffer( vim.current.buffer ) < 5:
self.parse_future = None
return return
filename = vim.current.buffer.name filename = vim.current.buffer.name
flags = self.flags.FlagsForFile( filename )
if not flags:
self.parse_future = None
return
self.parse_future = self.completer.UpdateTranslationUnitAsync( self.parse_future = self.completer.UpdateTranslationUnitAsync(
filename, filename,
self.GetUnsavedFilesVector(), self.GetUnsavedFilesVector(),
self.flags.FlagsForFile( filename ) ) flags )
def DiagnosticsForCurrentFileReady( self ): def DiagnosticsForCurrentFileReady( self ):

View File

@ -46,9 +46,13 @@ class Flags( object ):
flags_module = self._FlagsModuleForFile( filename ) flags_module = self._FlagsModuleForFile( filename )
if not flags_module: if not flags_module:
vimsupport.PostVimMessage( NO_OPTIONS_FILENAME_MESSAGE ) vimsupport.PostVimMessage( NO_OPTIONS_FILENAME_MESSAGE )
return ycm_core.StringVec() return None
results = flags_module.FlagsForFile( filename ) results = flags_module.FlagsForFile( filename )
if not results.get( 'flags_ready', True ):
return None
results[ 'flags' ] += self.special_clang_flags results[ 'flags' ] += self.special_clang_flags
sanitized_flags = _SanitizeFlags( results[ 'flags' ] ) sanitized_flags = _SanitizeFlags( results[ 'flags' ] )