vim/YouCompleteMe
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

No more threading in completers!

Browse Source 
The server is multi-threaded and will spawn a new thread for each new request.
Thus, the completers need not manage their own threads or even provide async
APIs; we _want_ them to block because now were implementing the request-response
networking API.

The client gets the async API through the network (i.e., it can do something
else while the request is pending).
This commit is contained in:
Strahinja Val Markovic 2013-09-30 13:40:25 -07:00
parent e38d145a47
commit 6c53bad58f
21 changed files with 189 additions and 1136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

343
cpp/ycm/ClangCompleter/ClangCompleter.cpp
View File

@ -1,4 +1,4 @@
// Copyright (C) 2011, 2012 Strahinja Val Markovic <val@markovic.io> // Copyright (C) 2011-2013 Strahinja Val Markovic <val@markovic.io>
// //
// This file is part of YouCompleteMe. // This file is part of YouCompleteMe.
// //
@ -28,43 +28,17 @@
#include "ClangUtils.h" #include "ClangUtils.h"
#include <clang-c/Index.h> #include <clang-c/Index.h>
#include <boost/make_shared.hpp> #include <boost/shared_ptr.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/cxx11/any_of.hpp>
using boost::algorithm::any_of;
using boost::algorithm::is_upper;
using boost::bind;
using boost::cref;
using boost::function;
using boost::lock_guard;
using boost::make_shared;
using boost::mutex;
using boost::packaged_task;
using boost::ref;
using boost::shared_lock;
using boost::shared_mutex;
using boost::shared_ptr; using boost::shared_ptr;
using boost::thread;
using boost::thread_interrupted;
using boost::try_to_lock_t;
using boost::unique_future;
using boost::unique_lock;
using boost::unordered_map; using boost::unordered_map;
namespace YouCompleteMe { namespace YouCompleteMe {
extern const unsigned int MAX_ASYNC_THREADS;
extern const unsigned int MIN_ASYNC_THREADS;
ClangCompleter::ClangCompleter() ClangCompleter::ClangCompleter()
: clang_index_( clang_createIndex( 0, 0 ) ), : clang_index_( clang_createIndex( 0, 0 ) ),
translation_unit_store_( clang_index_ ), translation_unit_store_( clang_index_ ) {
candidate_repository_( CandidateRepository::Instance() ),
threading_enabled_( false ),
time_to_die_( false ),
clang_data_ready_( false ) {
// The libclang docs don't say what is the default value for crash recovery. // The libclang docs don't say what is the default value for crash recovery.
// I'm pretty sure it's turned on by default, but I'm not going to take any // I'm pretty sure it's turned on by default, but I'm not going to take any
// chances. // chances.
@ -73,35 +47,18 @@ ClangCompleter::ClangCompleter()
ClangCompleter::~ClangCompleter() { ClangCompleter::~ClangCompleter() {
{
unique_lock< shared_mutex > lock( time_to_die_mutex_ );
time_to_die_ = true;
}
sorting_threads_.interrupt_all();
sorting_threads_.join_all();
if ( clang_thread_ ) {
clang_thread_->interrupt();
clang_thread_->join();
}
// We need to destroy all TUs before calling clang_disposeIndex because // We need to destroy all TUs before calling clang_disposeIndex because
// the translation units need to be destroyed before the index is destroyed. // the translation units need to be destroyed before the index is destroyed.
// Technically, a thread could still be holding onto a shared_ptr<TU> object
// when we destroy the clang index, but since we're shutting down, we don't
// really care.
// In practice, this situation shouldn't happen because the server threads are
// Python deamon threads and will all be killed before the main thread exits.
translation_unit_store_.RemoveAll(); translation_unit_store_.RemoveAll();
clang_disposeIndex( clang_index_ ); clang_disposeIndex( clang_index_ );
} }
// We need this mostly so that we can not use it in tests. Apparently the
// GoogleTest framework goes apeshit on us (on some platforms, in some
// occasions) if we enable threads by default.
void ClangCompleter::EnableThreading() {
threading_enabled_ = true;
InitThreads();
}
std::vector< Diagnostic > ClangCompleter::DiagnosticsForFile( std::vector< Diagnostic > ClangCompleter::DiagnosticsForFile(
std::string filename ) { std::string filename ) {
shared_ptr< TranslationUnit > unit = translation_unit_store_.Get( filename ); shared_ptr< TranslationUnit > unit = translation_unit_store_.Get( filename );
@ -156,30 +113,6 @@ void ClangCompleter::UpdateTranslationUnit(
} }
Future< void > ClangCompleter::UpdateTranslationUnitAsync(
std::string filename,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags ) {
function< void() > functor =
boost::bind( &ClangCompleter::UpdateTranslationUnit,
boost::ref( *this ),
boost::move( filename ),
boost::move( unsaved_files ),
boost::move( flags ) );
shared_ptr< ClangPackagedTask > clang_packaged_task =
make_shared< ClangPackagedTask >();
clang_packaged_task->parsing_task_ = packaged_task< void >(
boost::move( functor ) );
unique_future< void > future =
clang_packaged_task->parsing_task_.get_future();
clang_task_.Set( clang_packaged_task );
return Future< void >( boost::move( future ) );
}
std::vector< CompletionData > std::vector< CompletionData >
ClangCompleter::CandidatesForLocationInFile( ClangCompleter::CandidatesForLocationInFile(
std::string filename, std::string filename,
@ -199,56 +132,6 @@ ClangCompleter::CandidatesForLocationInFile(
} }
Future< AsyncCompletions >
ClangCompleter::CandidatesForQueryAndLocationInFileAsync(
std::string query,
std::string filename,
int line,
int column,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags ) {
// TODO: throw exception when threading is not enabled and this is called
if ( !threading_enabled_ )
return Future< AsyncCompletions >();
bool skip_clang_result_cache = ShouldSkipClangResultCache( query,
line,
column );
if ( skip_clang_result_cache ) {
// The clang thread is busy, return nothing
if ( UpdatingTranslationUnit( filename ) )
return Future< AsyncCompletions >();
{
lock_guard< mutex > lock( clang_data_ready_mutex_ );
clang_data_ready_ = false;
}
// Needed to "reset" the sorting threads to the start of their loop. This
// way any threads blocking on a read in sorting_task_.Get() are reset to
// wait on the clang_data_ready_condition_variable_.
sorting_threads_.interrupt_all();
}
// the sorting task needs to be set before the clang task (if any) just in
// case the clang task finishes (and therefore notifies a sorting thread to
// consume a sorting task) before the sorting task is set
unique_future< AsyncCompletions > future;
CreateSortingTask( query, future );
if ( skip_clang_result_cache ) {
CreateClangTask( boost::move( filename ),
line,
column,
boost::move( unsaved_files ),
boost::move( flags ) );
}
return Future< AsyncCompletions >( boost::move( future ) );
}
Location ClangCompleter::GetDeclarationLocation( Location ClangCompleter::GetDeclarationLocation(
std::string filename, std::string filename,
int line, int line,
@ -283,217 +166,9 @@ Location ClangCompleter::GetDefinitionLocation(
} }
void ClangCompleter::DeleteCachesForFileAsync( std::string filename ) { void ClangCompleter::DeleteCachesForFile( std::string filename ) {
file_cache_delete_stack_.Push( filename );
}
void ClangCompleter::DeleteCaches() {
std::vector< std::string > filenames;
if ( !file_cache_delete_stack_.PopAllNoWait( filenames ) )
return;
foreach( const std::string & filename, filenames ) {
translation_unit_store_.Remove( filename ); translation_unit_store_.Remove( filename );
} }
}
bool ClangCompleter::ShouldSkipClangResultCache( const std::string &query,
int line,
int column ) {
// We need query.empty() in addition to the second check because if we don't
// have it, then we have a problem in the following situation:
// The user has a variable 'foo' of type 'A' and a variable 'bar' of type 'B'.
// He then types in 'foo.' and we store the clang results and also return
// them. The user then deletes that code and types in 'bar.'. Without the
// query.empty() check we would return the results from the cache here (it's
// the same line and column!) which would be incorrect.
return query.empty() || latest_clang_results_
.NewPositionDifferentFromStoredPosition( line, column );
}
// Copy-ctor for unique_future is private in C++03 mode so we need to take it as
// an out param
void ClangCompleter::CreateSortingTask(
const std::string &query,
unique_future< AsyncCompletions > &future ) {
// Careful! The code in this function may burn your eyes.
function< CompletionDatas( const CompletionDatas & ) >
sort_candidates_for_query_functor =
boost::bind( &ClangCompleter::SortCandidatesForQuery,
boost::ref( *this ),
query,
_1 );
function< CompletionDatas() > operate_on_completion_data_functor =
boost::bind( &ClangResultsCache::OperateOnCompletionDatas< CompletionDatas >,
boost::cref( latest_clang_results_ ),
boost::move( sort_candidates_for_query_functor ) );
shared_ptr< packaged_task< AsyncCompletions > > task =
boost::make_shared< packaged_task< AsyncCompletions > >(
boost::bind( ReturnValueAsShared< std::vector< CompletionData > >,
boost::move( operate_on_completion_data_functor ) ) );
future = task->get_future();
sorting_task_.Set( task );
}
void ClangCompleter::CreateClangTask(
std::string filename,
int line,
int column,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags ) {
latest_clang_results_.ResetWithNewLineAndColumn( line, column );
function< CompletionDatas() > candidates_for_location_functor =
boost::bind( &ClangCompleter::CandidatesForLocationInFile,
boost::ref( *this ),
boost::move( filename ),
line,
column,
boost::move( unsaved_files ),
boost::move( flags ) );
shared_ptr< ClangPackagedTask > clang_packaged_task =
make_shared< ClangPackagedTask >();
clang_packaged_task->completions_task_ =
packaged_task< AsyncCompletions >(
boost::bind( ReturnValueAsShared< std::vector< CompletionData > >,
boost::move( candidates_for_location_functor ) ) );
clang_task_.Set( clang_packaged_task );
}
std::vector< CompletionData > ClangCompleter::SortCandidatesForQuery(
const std::string &query,
const std::vector< CompletionData > &completion_datas ) {
Bitset query_bitset = LetterBitsetFromString( query );
bool query_has_uppercase_letters = any_of( query, is_upper() );
std::vector< const Candidate * > repository_candidates =
candidate_repository_.GetCandidatesForStrings( completion_datas );
std::vector< ResultAnd< CompletionData * > > data_and_results;
for ( uint i = 0; i < repository_candidates.size(); ++i ) {
const Candidate *candidate = repository_candidates[ i ];
if ( !candidate->MatchesQueryBitset( query_bitset ) )
continue;
Result result = candidate->QueryMatchResult( query,
query_has_uppercase_letters );
if ( result.IsSubsequence() ) {
ResultAnd< CompletionData * > data_and_result( &completion_datas[ i ],
result );
data_and_results.push_back( boost::move( data_and_result ) );
}
}
std::sort( data_and_results.begin(), data_and_results.end() );
std::vector< CompletionData > sorted_completion_datas;
sorted_completion_datas.reserve( data_and_results.size() );
foreach ( const ResultAnd< CompletionData * > &data_and_result,
data_and_results ) {
sorted_completion_datas.push_back( *data_and_result.extra_object_ );
}
return sorted_completion_datas;
}
void ClangCompleter::InitThreads() {
int threads_to_create =
std::max( MIN_ASYNC_THREADS,
std::min( MAX_ASYNC_THREADS, thread::hardware_concurrency() ) );
for ( int i = 0; i < threads_to_create; ++i ) {
sorting_threads_.create_thread( bind( &ClangCompleter::SortingThreadMain,
boost::ref( *this ) ) );
}
clang_thread_.reset( new thread( &ClangCompleter::ClangThreadMain,
boost::ref( *this ) ) );
}
void ClangCompleter::ClangThreadMain() {
while ( true ) {
try {
shared_ptr< ClangPackagedTask > task = clang_task_.Get();
bool has_completions_task = task->completions_task_.valid();
if ( has_completions_task )
task->completions_task_();
else
task->parsing_task_();
if ( !has_completions_task ) {
DeleteCaches();
continue;
}
unique_future< AsyncCompletions > future =
task->completions_task_.get_future();
latest_clang_results_.SetCompletionDatas( *future.get() );
{
lock_guard< mutex > lock( clang_data_ready_mutex_ );
clang_data_ready_ = true;
}
clang_data_ready_condition_variable_.notify_all();
} catch ( thread_interrupted & ) {
shared_lock< shared_mutex > lock( time_to_die_mutex_ );
if ( time_to_die_ )
return;
// else do nothing and re-enter the loop
}
}
}
void ClangCompleter::SortingThreadMain() {
while ( true ) {
try {
{
unique_lock< mutex > lock( clang_data_ready_mutex_ );
while ( !clang_data_ready_ ) {
clang_data_ready_condition_variable_.wait( lock );
}
}
shared_ptr< packaged_task< AsyncCompletions > > task =
sorting_task_.Get();
( *task )();
} catch ( thread_interrupted & ) {
shared_lock< shared_mutex > lock( time_to_die_mutex_ );
if ( time_to_die_ )
return;
// else do nothing and re-enter the loop
}
}
}
} // namespace YouCompleteMe } // namespace YouCompleteMe

106
cpp/ycm/ClangCompleter/ClangCompleter.h
View File

@ -23,13 +23,9 @@
#include "Future.h" #include "Future.h"
#include "UnsavedFile.h" #include "UnsavedFile.h"
#include "Diagnostic.h" #include "Diagnostic.h"
#include "ClangResultsCache.h"
#include "TranslationUnitStore.h" #include "TranslationUnitStore.h"
#include <boost/utility.hpp> #include <boost/utility.hpp>
#include <boost/thread/future.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/unordered_map.hpp>
#include <string> #include <string>
@ -37,19 +33,12 @@ typedef struct CXTranslationUnitImpl *CXTranslationUnit;
namespace YouCompleteMe { namespace YouCompleteMe {
class CandidateRepository;
class TranslationUnit; class TranslationUnit;
struct CompletionData; struct CompletionData;
struct Location; struct Location;
typedef std::vector< CompletionData > CompletionDatas; typedef std::vector< CompletionData > CompletionDatas;
typedef boost::shared_ptr< CompletionDatas > AsyncCompletions;
typedef boost::unordered_map < std::string,
boost::shared_ptr <
std::vector< std::string > > > FlagsForFile;
// TODO: document that all filename parameters must be absolute paths // TODO: document that all filename parameters must be absolute paths
class ClangCompleter : boost::noncopyable { class ClangCompleter : boost::noncopyable {
@ -57,8 +46,6 @@ public:
ClangCompleter(); ClangCompleter();
~ClangCompleter(); ~ClangCompleter();
void EnableThreading();
std::vector< Diagnostic > DiagnosticsForFile( std::string filename ); std::vector< Diagnostic > DiagnosticsForFile( std::string filename );
bool UpdatingTranslationUnit( const std::string &filename ); bool UpdatingTranslationUnit( const std::string &filename );
@ -68,20 +55,11 @@ public:
// need to ensure we own the memory. // need to ensure we own the memory.
// TODO: Change some of these params back to const ref where possible after we // TODO: Change some of these params back to const ref where possible after we
// get the server up. // get the server up.
// TODO: Remove the async methods and the threads when the server is ready.
// Public because of unit tests (gtest is not very thread-friendly)
void UpdateTranslationUnit( void UpdateTranslationUnit(
std::string filename, std::string filename,
std::vector< UnsavedFile > unsaved_files, std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags ); std::vector< std::string > flags );
Future< void > UpdateTranslationUnitAsync(
std::string filename,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags );
// Public because of unit tests (gtest is not very thread-friendly)
std::vector< CompletionData > CandidatesForLocationInFile( std::vector< CompletionData > CandidatesForLocationInFile(
std::string filename, std::string filename,
int line, int line,
@ -89,14 +67,6 @@ public:
std::vector< UnsavedFile > unsaved_files, std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags ); std::vector< std::string > flags );
Future< AsyncCompletions > CandidatesForQueryAndLocationInFileAsync(
std::string query,
std::string filename,
int line,
int column,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags );
Location GetDeclarationLocation( Location GetDeclarationLocation(
std::string filename, std::string filename,
int line, int line,
@ -111,55 +81,10 @@ public:
std::vector< UnsavedFile > unsaved_files, std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags ); std::vector< std::string > flags );
void DeleteCachesForFileAsync( std::string filename ); void DeleteCachesForFile( std::string filename );
private: private:
void DeleteCaches();
// This is basically a union. Only one of the two tasks is set to something
// valid, the other task is invalid. Which one is valid depends on the caller.
struct ClangPackagedTask {
boost::packaged_task< AsyncCompletions > completions_task_;
boost::packaged_task< void > parsing_task_;
};
typedef ConcurrentLatestValue <
boost::shared_ptr <
boost::packaged_task< AsyncCompletions > > > LatestSortingTask;
typedef ConcurrentLatestValue <
boost::shared_ptr< ClangPackagedTask > > LatestClangTask;
typedef ConcurrentStack< std::string > FileCacheDeleteStack;
bool ShouldSkipClangResultCache( const std::string &query,
int line,
int column );
void CreateSortingTask( const std::string &query,
boost::unique_future< AsyncCompletions > &future );
// NOTE: params are taken by value on purpose! With a C++11 compiler we can
// avoid internal copies if params are taken by value (move ctors FTW)
void CreateClangTask(
std::string filename,
int line,
int column,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags );
std::vector< CompletionData > SortCandidatesForQuery(
const std::string &query,
const std::vector< CompletionData > &completion_datas );
void InitThreads();
void ClangThreadMain();
void SortingThreadMain();
///////////////////////////// /////////////////////////////
// PRIVATE MEMBER VARIABLES // PRIVATE MEMBER VARIABLES
///////////////////////////// /////////////////////////////
@ -167,35 +92,6 @@ private:
CXIndex clang_index_; CXIndex clang_index_;
TranslationUnitStore translation_unit_store_; TranslationUnitStore translation_unit_store_;
CandidateRepository &candidate_repository_;
bool threading_enabled_;
// TODO: use boost.atomic for time_to_die_
bool time_to_die_;
boost::shared_mutex time_to_die_mutex_;
// TODO: use boost.atomic for clang_data_ready_
bool clang_data_ready_;
boost::mutex clang_data_ready_mutex_;
boost::condition_variable clang_data_ready_condition_variable_;
ClangResultsCache latest_clang_results_;
FileCacheDeleteStack file_cache_delete_stack_;
// Unfortunately clang is not thread-safe so we need to be careful when we
// access it. Only one thread at a time is allowed to access any single
// translation unit. Currently we only use one thread to access clang and that
// is the thread represented by clang_thread_.
boost::scoped_ptr< boost::thread > clang_thread_;
boost::thread_group sorting_threads_;
mutable LatestClangTask clang_task_;
mutable LatestSortingTask sorting_task_;
}; };
} // namespace YouCompleteMe } // namespace YouCompleteMe

43
cpp/ycm/ClangCompleter/ClangResultsCache.cpp
View File

@ -1,43 +0,0 @@
// Copyright (C) 2011, 2012 Strahinja Val Markovic <val@markovic.io>
//
// This file is part of YouCompleteMe.
//
// YouCompleteMe is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// YouCompleteMe is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with YouCompleteMe. If not, see <http://www.gnu.org/licenses/>.
#include "ClangResultsCache.h"
#include "standard.h"
using boost::shared_mutex;
using boost::shared_lock;
using boost::unique_lock;
namespace YouCompleteMe {
bool ClangResultsCache::NewPositionDifferentFromStoredPosition( int new_line,
int new_colum )
const {
shared_lock< shared_mutex > reader_lock( access_mutex_ );
return line_ != new_line || column_ != new_colum;
}
void ClangResultsCache::ResetWithNewLineAndColumn( int new_line,
int new_colum ) {
unique_lock< shared_mutex > reader_lock( access_mutex_ );
line_ = new_line;
column_ = new_colum;
completion_datas_.clear();
}
} // namespace YouCompleteMe

84
cpp/ycm/ClangCompleter/ClangResultsCache.h
View File

@ -1,84 +0,0 @@
// Copyright (C) 2011, 2012 Strahinja Val Markovic <val@markovic.io>
//
// This file is part of YouCompleteMe.
//
// YouCompleteMe is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// YouCompleteMe is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with YouCompleteMe. If not, see <http://www.gnu.org/licenses/>.
#ifndef CLANGRESULTSCACHE_H_REUWM3RU
#define CLANGRESULTSCACHE_H_REUWM3RU
#include "CompletionData.h"
#include <vector>
#include <boost/thread/locks.hpp>
#include <boost/thread/shared_mutex.hpp>
#include <boost/function.hpp>
#include <boost/config.hpp>
#include <boost/utility.hpp>
namespace YouCompleteMe {
struct CompletionData;
class ClangResultsCache : boost::noncopyable {
public:
ClangResultsCache() : line_( -1 ), column_( -1 ) {}
bool NewPositionDifferentFromStoredPosition( int new_line, int new_colum )
const;
void ResetWithNewLineAndColumn( int new_line, int new_colum );
void SetCompletionDatas(
const std::vector< CompletionData > new_completions ) {
completion_datas_ = new_completions;
}
#ifndef BOOST_NO_RVALUE_REFERENCES
# ifdef __clang__
# pragma clang diagnostic push
# pragma clang diagnostic ignored "-Wc++98-compat"
# endif //#ifdef __clang__
void SetCompletionDatas( std::vector< CompletionData > &&new_completions ) {
completion_datas_ = new_completions;
}
# ifdef __clang__
# pragma clang diagnostic pop
# endif //#ifdef __clang__
#endif //#ifndef BOOST_NO_RVALUE_REFERENCES
template< typename T >
T OperateOnCompletionDatas(
boost::function< T( const std::vector< CompletionData >& ) > operation )
const {
boost::shared_lock< boost::shared_mutex > reader_lock( access_mutex_ );
return operation( completion_datas_ );
}
private:
int line_;
int column_;
std::vector< CompletionData > completion_datas_;
mutable boost::shared_mutex access_mutex_;
};
} // namespace YouCompleteMe
#endif /* end of include guard: CLANGRESULTSCACHE_H_REUWM3RU */

7
cpp/ycm/ClangCompleter/TranslationUnitStore.cpp
View File

@ -38,14 +38,17 @@ std::size_t HashForFlags( const std::vector< std::string > &flags ) {
} // unnamed namespace } // unnamed namespace
TranslationUnitStore::TranslationUnitStore( CXIndex clang_index ) TranslationUnitStore::TranslationUnitStore( CXIndex clang_index )
: clang_index_( clang_index ) { : clang_index_( clang_index ) {
} }
TranslationUnitStore::~TranslationUnitStore() { TranslationUnitStore::~TranslationUnitStore() {
RemoveAll(); RemoveAll();
} }
shared_ptr< TranslationUnit > TranslationUnitStore::GetOrCreate( shared_ptr< TranslationUnit > TranslationUnitStore::GetOrCreate(
const std::string &filename, const std::string &filename,
const std::vector< UnsavedFile > &unsaved_files, const std::vector< UnsavedFile > &unsaved_files,
@ -104,24 +107,28 @@ shared_ptr< TranslationUnit > TranslationUnitStore::GetOrCreate(
return unit; return unit;
} }
shared_ptr< TranslationUnit > TranslationUnitStore::Get( shared_ptr< TranslationUnit > TranslationUnitStore::Get(
const std::string &filename ) { const std::string &filename ) {
lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ ); lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ );
return GetNoLock( filename ); return GetNoLock( filename );
} }
bool TranslationUnitStore::Remove( const std::string &filename ) { bool TranslationUnitStore::Remove( const std::string &filename ) {
lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ ); lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ );
Erase( filename_to_flags_hash_, filename ); Erase( filename_to_flags_hash_, filename );
return Erase( filename_to_translation_unit_, filename ); return Erase( filename_to_translation_unit_, filename );
} }
void TranslationUnitStore::RemoveAll() { void TranslationUnitStore::RemoveAll() {
lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ ); lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ );
filename_to_translation_unit_.clear(); filename_to_translation_unit_.clear();
filename_to_flags_hash_.clear(); filename_to_flags_hash_.clear();
} }
shared_ptr< TranslationUnit > TranslationUnitStore::GetNoLock( shared_ptr< TranslationUnit > TranslationUnitStore::GetNoLock(
const std::string &filename ) { const std::string &filename ) {
return FindWithDefault( filename_to_translation_unit_, return FindWithDefault( filename_to_translation_unit_,

152
cpp/ycm/IdentifierCompleter.cpp
View File

@ -23,62 +23,16 @@
#include "Result.h" #include "Result.h"
#include "Utils.h" #include "Utils.h"
#include <boost/bind.hpp>
#include <boost/make_shared.hpp>
#include <algorithm> #include <algorithm>
using boost::packaged_task;
using boost::unique_future;
using boost::shared_ptr;
using boost::thread;
namespace YouCompleteMe { namespace YouCompleteMe {
typedef boost::function< std::vector< std::string >() >
FunctionReturnsStringVector;
IdentifierCompleter::IdentifierCompleter() {}
extern const unsigned int MAX_ASYNC_THREADS = 4;
extern const unsigned int MIN_ASYNC_THREADS = 2;
namespace {
void QueryThreadMain(
IdentifierCompleter::LatestQueryTask &latest_query_task ) {
while ( true ) {
try {
( *latest_query_task.Get() )();
} catch ( boost::thread_interrupted & ) {
return;
}
}
}
void BufferIdentifiersThreadMain(
IdentifierCompleter::BufferIdentifiersTaskStack
&buffer_identifiers_task_stack ) {
while ( true ) {
try {
( *buffer_identifiers_task_stack.Pop() )();
} catch ( boost::thread_interrupted & ) {
return;
}
}
}
} // unnamed namespace
IdentifierCompleter::IdentifierCompleter()
: threading_enabled_( false ) {
}
IdentifierCompleter::IdentifierCompleter( IdentifierCompleter::IdentifierCompleter(
const std::vector< std::string > &candidates ) const std::vector< std::string > &candidates ) {
: threading_enabled_( false ) {
identifier_database_.AddIdentifiers( candidates, "", "" ); identifier_database_.AddIdentifiers( candidates, "", "" );
} }
@ -86,31 +40,11 @@ IdentifierCompleter::IdentifierCompleter(
IdentifierCompleter::IdentifierCompleter( IdentifierCompleter::IdentifierCompleter(
const std::vector< std::string > &candidates, const std::vector< std::string > &candidates,
const std::string &filetype, const std::string &filetype,
const std::string &filepath ) const std::string &filepath ) {
: threading_enabled_( false ) {
identifier_database_.AddIdentifiers( candidates, filetype, filepath ); identifier_database_.AddIdentifiers( candidates, filetype, filepath );
} }
IdentifierCompleter::~IdentifierCompleter() {
query_threads_.interrupt_all();
query_threads_.join_all();
if ( buffer_identifiers_thread_ ) {
buffer_identifiers_thread_->interrupt();
buffer_identifiers_thread_->join();
}
}
// We need this mostly so that we can not use it in tests. Apparently the
// GoogleTest framework goes apeshit on us if we enable threads by default.
void IdentifierCompleter::EnableThreading() {
threading_enabled_ = true;
InitThreads();
}
void IdentifierCompleter::AddIdentifiersToDatabase( void IdentifierCompleter::AddIdentifiersToDatabase(
const std::vector< std::string > &new_candidates, const std::vector< std::string > &new_candidates,
const std::string &filetype, const std::string &filetype,
@ -130,22 +64,6 @@ void IdentifierCompleter::AddIdentifiersToDatabaseFromTagFiles(
} }
void IdentifierCompleter::AddIdentifiersToDatabaseFromTagFilesAsync(
std::vector< std::string > absolute_paths_to_tag_files ) {
// TODO: throw exception when threading is not enabled and this is called
if ( !threading_enabled_ )
return;
boost::function< void() > functor =
boost::bind( &IdentifierCompleter::AddIdentifiersToDatabaseFromTagFiles,
boost::ref( *this ),
boost::move( absolute_paths_to_tag_files ) );
buffer_identifiers_task_stack_.Push(
boost::make_shared< packaged_task< void > >( boost::move( functor ) ) );
}
void IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer( void IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer(
const std::string &buffer_contents, const std::string &buffer_contents,
const std::string &filetype, const std::string &filetype,
@ -165,28 +83,6 @@ void IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer(
} }
void IdentifierCompleter::AddIdentifiersToDatabaseFromBufferAsync(
std::string buffer_contents,
std::string filetype,
std::string filepath,
bool collect_from_comments_and_strings ) {
// TODO: throw exception when threading is not enabled and this is called
if ( !threading_enabled_ )
return;
boost::function< void() > functor =
boost::bind( &IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer,
boost::ref( *this ),
boost::move( buffer_contents ),
boost::move( filetype ),
boost::move( filepath ),
collect_from_comments_and_strings );
buffer_identifiers_task_stack_.Push(
boost::make_shared< packaged_task< void > >( boost::move( functor ) ) );
}
std::vector< std::string > IdentifierCompleter::CandidatesForQuery( std::vector< std::string > IdentifierCompleter::CandidatesForQuery(
const std::string &query ) const { const std::string &query ) const {
return CandidatesForQueryAndType( query, "" ); return CandidatesForQueryAndType( query, "" );
@ -209,46 +105,4 @@ std::vector< std::string > IdentifierCompleter::CandidatesForQueryAndType(
} }
Future< AsyncResults > IdentifierCompleter::CandidatesForQueryAndTypeAsync(
const std::string &query,
const std::string &filetype ) const {
// TODO: throw exception when threading is not enabled and this is called
if ( !threading_enabled_ )
return Future< AsyncResults >();
FunctionReturnsStringVector functor =
boost::bind( &IdentifierCompleter::CandidatesForQueryAndType,
boost::cref( *this ),
query,
filetype );
QueryTask task =
boost::make_shared< packaged_task< AsyncResults > >(
boost::bind( ReturnValueAsShared< std::vector< std::string > >,
boost::move( functor ) ) );
unique_future< AsyncResults > future = task->get_future();
latest_query_task_.Set( task );
return Future< AsyncResults >( boost::move( future ) );
}
void IdentifierCompleter::InitThreads() {
int query_threads_to_create =
std::max( MIN_ASYNC_THREADS,
std::min( MAX_ASYNC_THREADS, thread::hardware_concurrency() ) );
for ( int i = 0; i < query_threads_to_create; ++i ) {
query_threads_.create_thread(
boost::bind( QueryThreadMain,
boost::ref( latest_query_task_ ) ) );
}
buffer_identifiers_thread_.reset(
new boost::thread( BufferIdentifiersThreadMain,
boost::ref( buffer_identifiers_task_stack_ ) ) );
}
} // namespace YouCompleteMe } // namespace YouCompleteMe

43
cpp/ycm/IdentifierCompleter.h
View File

@ -36,8 +36,6 @@ namespace YouCompleteMe {
class Candidate; class Candidate;
typedef boost::shared_ptr< std::vector< std::string > > AsyncResults;
class IdentifierCompleter : boost::noncopyable { class IdentifierCompleter : boost::noncopyable {
public: public:
@ -47,10 +45,6 @@ public:
const std::string &filetype, const std::string &filetype,
const std::string &filepath ); const std::string &filepath );
~IdentifierCompleter();
void EnableThreading();
void AddIdentifiersToDatabase( void AddIdentifiersToDatabase(
const std::vector< std::string > &new_candidates, const std::vector< std::string > &new_candidates,
const std::string &filetype, const std::string &filetype,
@ -59,25 +53,12 @@ public:
void AddIdentifiersToDatabaseFromTagFiles( void AddIdentifiersToDatabaseFromTagFiles(
const std::vector< std::string > &absolute_paths_to_tag_files ); const std::vector< std::string > &absolute_paths_to_tag_files );
// NOTE: params are taken by value on purpose!
void AddIdentifiersToDatabaseFromTagFilesAsync(
std::vector< std::string > absolute_paths_to_tag_files );
void AddIdentifiersToDatabaseFromBuffer( void AddIdentifiersToDatabaseFromBuffer(
const std::string &buffer_contents, const std::string &buffer_contents,
const std::string &filetype, const std::string &filetype,
const std::string &filepath, const std::string &filepath,
bool collect_from_comments_and_strings ); bool collect_from_comments_and_strings );
// NOTE: params are taken by value on purpose! With a C++11 compiler we can
// avoid an expensive copy of buffer_contents if the param is taken by value
// (move ctors FTW)
void AddIdentifiersToDatabaseFromBufferAsync(
std::string buffer_contents,
std::string filetype,
std::string filepath,
bool collect_from_comments_and_strings );
// Only provided for tests! // Only provided for tests!
std::vector< std::string > CandidatesForQuery( std::vector< std::string > CandidatesForQuery(
const std::string &query ) const; const std::string &query ) const;
@ -86,37 +67,13 @@ public:
const std::string &query, const std::string &query,
const std::string &filetype ) const; const std::string &filetype ) const;
Future< AsyncResults > CandidatesForQueryAndTypeAsync(
const std::string &query,
const std::string &filetype ) const;
typedef boost::shared_ptr <
boost::packaged_task< AsyncResults > > QueryTask;
typedef ConcurrentLatestValue< QueryTask > LatestQueryTask;
typedef ConcurrentStack< VoidTask > BufferIdentifiersTaskStack;
private: private:
void InitThreads();
///////////////////////////// /////////////////////////////
// PRIVATE MEMBER VARIABLES // PRIVATE MEMBER VARIABLES
///////////////////////////// /////////////////////////////
IdentifierDatabase identifier_database_; IdentifierDatabase identifier_database_;
bool threading_enabled_;
boost::thread_group query_threads_;
boost::scoped_ptr< boost::thread > buffer_identifiers_thread_;
mutable LatestQueryTask latest_query_task_;
BufferIdentifiersTaskStack buffer_identifiers_task_stack_;
}; };
} // namespace YouCompleteMe } // namespace YouCompleteMe

18
cpp/ycm/tests/ClangCompleter/ClangCompleter_test.cpp
View File

@ -43,24 +43,6 @@ TEST( ClangCompleterTest, CandidatesForLocationInFile ) {
} }
TEST( ClangCompleterTest, CandidatesForQueryAndLocationInFileAsync ) {
ClangCompleter completer;
completer.EnableThreading();
Future< AsyncCompletions > completions_future =
completer.CandidatesForQueryAndLocationInFileAsync(
"",
PathToTestFile( "basic.cpp" ).string(),
11,
7,
std::vector< UnsavedFile >(),
std::vector< std::string >() );
completions_future.Wait();
EXPECT_TRUE( !completions_future.GetResults()->empty() );
}
TEST( ClangCompleterTest, GetDefinitionLocation ) { TEST( ClangCompleterTest, GetDefinitionLocation ) {
ClangCompleter completer; ClangCompleter completer;
std::string filename = PathToTestFile( "basic.cpp" ).string(); std::string filename = PathToTestFile( "basic.cpp" ).string();

37
cpp/ycm/ycm_core.cpp
View File

@ -60,36 +60,24 @@ BOOST_PYTHON_MODULE(ycm_core)
def( "YcmCoreVersion", YcmCoreVersion ); def( "YcmCoreVersion", YcmCoreVersion );
class_< IdentifierCompleter, boost::noncopyable >( "IdentifierCompleter" ) class_< IdentifierCompleter, boost::noncopyable >( "IdentifierCompleter" )
.def( "EnableThreading", &IdentifierCompleter::EnableThreading )
.def( "AddIdentifiersToDatabase", .def( "AddIdentifiersToDatabase",
&IdentifierCompleter::AddIdentifiersToDatabase ) &IdentifierCompleter::AddIdentifiersToDatabase )
.def( "AddIdentifiersToDatabaseFromTagFilesAsync", .def( "AddIdentifiersToDatabaseFromTagFiles",
&IdentifierCompleter::AddIdentifiersToDatabaseFromTagFilesAsync ) &IdentifierCompleter::AddIdentifiersToDatabaseFromTagFiles )
.def( "AddIdentifiersToDatabaseFromBufferAsync", .def( "AddIdentifiersToDatabaseFromBuffer",
&IdentifierCompleter::AddIdentifiersToDatabaseFromBufferAsync ) &IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer )
.def( "CandidatesForQueryAndTypeAsync", .def( "CandidatesForQueryAndType",
&IdentifierCompleter::CandidatesForQueryAndTypeAsync ); &IdentifierCompleter::CandidatesForQueryAndType );
// TODO: rename these *Vec classes to *Vector; don't forget the python file // TODO: rename these *Vec classes to *Vector; don't forget the python file
class_< std::vector< std::string >, class_< std::vector< std::string >,
boost::shared_ptr< std::vector< std::string > > >( "StringVec" ) boost::shared_ptr< std::vector< std::string > > >( "StringVec" )
.def( vector_indexing_suite< std::vector< std::string > >() ); .def( vector_indexing_suite< std::vector< std::string > >() );
class_< Future< AsyncResults > >( "FutureResults" )
.def( "ResultsReady", &Future< AsyncResults >::ResultsReady )
.def( "GetResults", &Future< AsyncResults >::GetResults );
class_< Future< void > >( "FutureVoid" )
.def( "ResultsReady", &Future< void >::ResultsReady )
.def( "GetResults", &Future< void >::GetResults );
#ifdef USE_CLANG_COMPLETER #ifdef USE_CLANG_COMPLETER
def( "ClangVersion", ClangVersion ); def( "ClangVersion", ClangVersion );
class_< Future< AsyncCompletions > >( "FutureCompletions" ) // TODO: We may not need this at all anymore. Look into it.
.def( "ResultsReady", &Future< AsyncCompletions >::ResultsReady )
.def( "GetResults", &Future< AsyncCompletions >::GetResults );
class_< Future< AsyncCompilationInfoForFile > >( class_< Future< AsyncCompilationInfoForFile > >(
"FutureCompilationInfoForFile" ) "FutureCompilationInfoForFile" )
.def( "ResultsReady", .def( "ResultsReady",
@ -116,17 +104,14 @@ BOOST_PYTHON_MODULE(ycm_core)
.def( vector_indexing_suite< std::vector< UnsavedFile > >() ); .def( vector_indexing_suite< std::vector< UnsavedFile > >() );
class_< ClangCompleter, boost::noncopyable >( "ClangCompleter" ) class_< ClangCompleter, boost::noncopyable >( "ClangCompleter" )
.def( "EnableThreading", &ClangCompleter::EnableThreading )
.def( "DiagnosticsForFile", &ClangCompleter::DiagnosticsForFile ) .def( "DiagnosticsForFile", &ClangCompleter::DiagnosticsForFile )
.def( "GetDeclarationLocation", &ClangCompleter::GetDeclarationLocation ) .def( "GetDeclarationLocation", &ClangCompleter::GetDeclarationLocation )
.def( "GetDefinitionLocation", &ClangCompleter::GetDefinitionLocation ) .def( "GetDefinitionLocation", &ClangCompleter::GetDefinitionLocation )
.def( "DeleteCachesForFileAsync", .def( "DeleteCachesForFile", &ClangCompleter::DeleteCachesForFile )
&ClangCompleter::DeleteCachesForFileAsync )
.def( "UpdatingTranslationUnit", &ClangCompleter::UpdatingTranslationUnit ) .def( "UpdatingTranslationUnit", &ClangCompleter::UpdatingTranslationUnit )
.def( "UpdateTranslationUnitAsync", .def( "UpdateTranslationUnit", &ClangCompleter::UpdateTranslationUnit )
&ClangCompleter::UpdateTranslationUnitAsync ) .def( "CandidatesForLocationInFile",
.def( "CandidatesForQueryAndLocationInFileAsync", &ClangCompleter::CandidatesForLocationInFile );
&ClangCompleter::CandidatesForQueryAndLocationInFileAsync );
class_< CompletionData >( "CompletionData" ) class_< CompletionData >( "CompletionData" )
.def( "TextToInsertInBuffer", &CompletionData::TextToInsertInBuffer ) .def( "TextToInsertInBuffer", &CompletionData::TextToInsertInBuffer )

2
python/test_requirements.txt
View File

@ -1,3 +1,5 @@
flake8>=2.0 flake8>=2.0
mock>=1.0.1 mock>=1.0.1
nose>=1.3.0 nose>=1.3.0
PyHamcrest>=1.7.2

42
python/ycm/completers/all/identifier_completer.py
View File

@ -34,9 +34,8 @@ SYNTAX_FILENAME = 'YCM_PLACEHOLDER_FOR_SYNTAX'
class IdentifierCompleter( GeneralCompleter ): class IdentifierCompleter( GeneralCompleter ):
def __init__( self, user_options ): def __init__( self, user_options ):
super( IdentifierCompleter, self ).__init__( user_options ) super( IdentifierCompleter, self ).__init__( user_options )
self.completer = ycm_core.IdentifierCompleter() self._completer = ycm_core.IdentifierCompleter()
self.completer.EnableThreading() self._tags_file_last_mtime = defaultdict( int )
self.tags_file_last_mtime = defaultdict( int )
self._logger = logging.getLogger( __name__ ) self._logger = logging.getLogger( __name__ )
@ -44,11 +43,20 @@ class IdentifierCompleter( GeneralCompleter ):
return self.QueryLengthAboveMinThreshold( request_data ) return self.QueryLengthAboveMinThreshold( request_data )
def CandidatesForQueryAsync( self, request_data ): def ComputeCandidates( self, request_data ):
self.completions_future = self.completer.CandidatesForQueryAndTypeAsync( if not self.ShouldUseNow( request_data ):
return []
completions = self._completer.CandidatesForQueryAndType(
ToUtf8IfNeeded( utils.SanitizeQuery( request_data[ 'query' ] ) ), ToUtf8IfNeeded( utils.SanitizeQuery( request_data[ 'query' ] ) ),
ToUtf8IfNeeded( request_data[ 'filetypes' ][ 0 ] ) ) ToUtf8IfNeeded( request_data[ 'filetypes' ][ 0 ] ) )
completions = completions[ : MAX_IDENTIFIER_COMPLETIONS_RETURNED ]
completions = _RemoveSmallCandidates(
completions, self.user_options[ 'min_num_identifier_candidate_chars' ] )
return [ responses.BuildCompletionData( x ) for x in completions ]
def AddIdentifier( self, identifier, request_data ): def AddIdentifier( self, identifier, request_data ):
filetype = request_data[ 'filetypes' ][ 0 ] filetype = request_data[ 'filetypes' ][ 0 ]
@ -60,7 +68,7 @@ class IdentifierCompleter( GeneralCompleter ):
vector = ycm_core.StringVec() vector = ycm_core.StringVec()
vector.append( ToUtf8IfNeeded( identifier ) ) vector.append( ToUtf8IfNeeded( identifier ) )
self._logger.info( 'Adding ONE buffer identifier for file: %s', filepath ) self._logger.info( 'Adding ONE buffer identifier for file: %s', filepath )
self.completer.AddIdentifiersToDatabase( vector, self._completer.AddIdentifiersToDatabase( vector,
ToUtf8IfNeeded( filetype ), ToUtf8IfNeeded( filetype ),
ToUtf8IfNeeded( filepath ) ) ToUtf8IfNeeded( filepath ) )
@ -92,7 +100,7 @@ class IdentifierCompleter( GeneralCompleter ):
text = request_data[ 'file_data' ][ filepath ][ 'contents' ] text = request_data[ 'file_data' ][ filepath ][ 'contents' ]
self._logger.info( 'Adding buffer identifiers for file: %s', filepath ) self._logger.info( 'Adding buffer identifiers for file: %s', filepath )
self.completer.AddIdentifiersToDatabaseFromBufferAsync( self._completer.AddIdentifiersToDatabaseFromBuffer(
ToUtf8IfNeeded( text ), ToUtf8IfNeeded( text ),
ToUtf8IfNeeded( filetype ), ToUtf8IfNeeded( filetype ),
ToUtf8IfNeeded( filepath ), ToUtf8IfNeeded( filepath ),
@ -106,20 +114,20 @@ class IdentifierCompleter( GeneralCompleter ):
current_mtime = os.path.getmtime( tag_file ) current_mtime = os.path.getmtime( tag_file )
except: except:
continue continue
last_mtime = self.tags_file_last_mtime[ tag_file ] last_mtime = self._tags_file_last_mtime[ tag_file ]
# We don't want to repeatedly process the same file over and over; we only # We don't want to repeatedly process the same file over and over; we only
# process if it's changed since the last time we looked at it # process if it's changed since the last time we looked at it
if current_mtime <= last_mtime: if current_mtime <= last_mtime:
continue continue
self.tags_file_last_mtime[ tag_file ] = current_mtime self._tags_file_last_mtime[ tag_file ] = current_mtime
absolute_paths_to_tag_files.append( ToUtf8IfNeeded( tag_file ) ) absolute_paths_to_tag_files.append( ToUtf8IfNeeded( tag_file ) )
if not absolute_paths_to_tag_files: if not absolute_paths_to_tag_files:
return return
self.completer.AddIdentifiersToDatabaseFromTagFilesAsync( self._completer.AddIdentifiersToDatabaseFromTagFiles(
absolute_paths_to_tag_files ) absolute_paths_to_tag_files )
@ -129,7 +137,7 @@ class IdentifierCompleter( GeneralCompleter ):
keyword_vector.append( ToUtf8IfNeeded( keyword ) ) keyword_vector.append( ToUtf8IfNeeded( keyword ) )
filepath = SYNTAX_FILENAME + filetypes[ 0 ] filepath = SYNTAX_FILENAME + filetypes[ 0 ]
self.completer.AddIdentifiersToDatabase( keyword_vector, self._completer.AddIdentifiersToDatabase( keyword_vector,
ToUtf8IfNeeded( filetypes[ 0 ] ), ToUtf8IfNeeded( filetypes[ 0 ] ),
ToUtf8IfNeeded( filepath ) ) ToUtf8IfNeeded( filepath ) )
@ -151,18 +159,6 @@ class IdentifierCompleter( GeneralCompleter ):
self.AddPreviousIdentifier( request_data ) self.AddPreviousIdentifier( request_data )
def CandidatesFromStoredRequest( self ):
if not self.completions_future:
return []
completions = self.completions_future.GetResults()[
: MAX_IDENTIFIER_COMPLETIONS_RETURNED ]
completions = _RemoveSmallCandidates(
completions, self.user_options[ 'min_num_identifier_candidate_chars' ] )
return [ responses.BuildCompletionData( x ) for x in completions ]
def _PreviousIdentifier( min_num_completion_start_chars, request_data ): def _PreviousIdentifier( min_num_completion_start_chars, request_data ):
line_num = request_data[ 'line_num' ] line_num = request_data[ 'line_num' ]
column_num = request_data[ 'column_num' ] column_num = request_data[ 'column_num' ]

117
python/ycm/completers/completer.py
View File

@ -58,44 +58,21 @@ class Completer( object ):
and will NOT re-query your completer but will in fact provide fuzzy-search on and will NOT re-query your completer but will in fact provide fuzzy-search on
the candidate strings that were stored in the cache. the candidate strings that were stored in the cache.
CandidatesForQueryAsync() is the main entry point when the user types. For ComputeCandidates() is the main entry point when the user types. For
"foo.bar", the user query is "bar" and completions matching this string should "foo.bar", the user query is "bar" and completions matching this string should
be shown. The job of CandidatesForQueryAsync() is to merely initiate this be shown. It should return the list of candidates. The format of the result
request, which will hopefully be processed in a background thread. You may can be a list of strings or a more complicated list of dictionaries. Use
want to subclass ThreadedCompleter instead of Completer directly. ycm.server.responses.BuildCompletionData to build the detailed response. See
clang_completer.py to see how its used in practice.
AsyncCandidateRequestReady() is the function that is repeatedly polled until Again, you probably want to override ComputeCandidatesInner().
it returns True. If CandidatesForQueryAsync() started a background task of
collecting the required completions, AsyncCandidateRequestReady() would check
the state of that task and return False until it was completed.
CandidatesFromStoredRequest() should return the list of candidates. This is
what YCM calls after AsyncCandidateRequestReady() returns True. The format of
the result can be a list of strings or a more complicated list of
dictionaries. See ':h complete-items' for the format, and clang_completer.py
to see how its used in practice.
You also need to implement the SupportedFiletypes() function which should You also need to implement the SupportedFiletypes() function which should
return a list of strings, where the strings are Vim filetypes your completer return a list of strings, where the strings are Vim filetypes your completer
supports. supports.
clang_completer.py is a good example of a "complicated" completer that clang_completer.py is a good example of a "complicated" completer that A good
maintains its own internal cache and therefore directly overrides the "main" example of a simple completer is ultisnips_completer.py.
functions in the API instead of the *Inner versions. A good example of a
simple completer that does not do this is omni_completer.py.
If you're confident your completer doesn't need a background task (think
again, you probably do) because you can "certainly" furnish a response in
under 10ms, then you can perform your backend processing in a synchronous
fashion. You may also need to do this because of technical restrictions (much
like omni_completer.py has to do it because accessing Vim internals is not
thread-safe). But even if you're certain, still try to do the processing in a
background thread. Your completer is unlikely to be merged if it does not,
because synchronous processing will block Vim's GUI thread and that's a very,
VERY bad thing (so try not to do it!). Again, you may want to subclass
ThreadedCompleter instead of Completer directly; ThreadedCompleter will
abstract away the use of a background thread for you. See
threaded_completer.py.
The On* functions are provided for your convenience. They are called when The On* functions are provided for your convenience. They are called when
their specific events occur. For instance, the identifier completer collects their specific events occur. For instance, the identifier completer collects
@ -122,7 +99,7 @@ class Completer( object ):
self.triggers_for_filetype = TriggersForFiletype( self.triggers_for_filetype = TriggersForFiletype(
user_options[ 'semantic_triggers' ] ) user_options[ 'semantic_triggers' ] )
self.completions_future = None self.completions_future = None
self.completions_cache = None self._completions_cache = None
# It's highly likely you DON'T want to override this function but the *Inner # It's highly likely you DON'T want to override this function but the *Inner
@ -130,13 +107,13 @@ class Completer( object ):
def ShouldUseNow( self, request_data ): def ShouldUseNow( self, request_data ):
inner_says_yes = self.ShouldUseNowInner( request_data ) inner_says_yes = self.ShouldUseNowInner( request_data )
if not inner_says_yes: if not inner_says_yes:
self.completions_cache = None self._completions_cache = None
previous_results_were_empty = ( self.completions_cache and previous_results_were_empty = ( self._completions_cache and
self.completions_cache.CacheValid( self._completions_cache.CacheValid(
request_data[ 'line_num' ], request_data[ 'line_num' ],
request_data[ 'start_column' ] ) and request_data[ 'start_column' ] ) and
not self.completions_cache.raw_completions ) not self._completions_cache.raw_completions )
return inner_says_yes and not previous_results_were_empty return inner_says_yes and not previous_results_were_empty
@ -171,20 +148,28 @@ class Completer( object ):
# It's highly likely you DON'T want to override this function but the *Inner # It's highly likely you DON'T want to override this function but the *Inner
# version of it. # version of it.
def CandidatesForQueryAsync( self, request_data ): def ComputeCandidates( self, request_data ):
self.request_data = request_data if not self.ShouldUseNow( request_data ):
return []
if ( request_data[ 'query' ] and if ( request_data[ 'query' ] and
self.completions_cache and self._completions_cache and
self.completions_cache.CacheValid( request_data[ 'line_num' ], self._completions_cache.CacheValid( request_data[ 'line_num' ],
request_data[ 'start_column' ] ) ): request_data[ 'start_column' ] ) ):
self.completions_cache.filtered_completions = ( return self.FilterAndSortCandidates(
self.FilterAndSortCandidates( self._completions_cache.raw_completions,
self.completions_cache.raw_completions, request_data[ 'query' ] )
request_data[ 'query' ] ) )
else: else:
self.completions_cache = None self._completions_cache = CompletionsCache()
self.CandidatesForQueryAsyncInner( request_data ) self._completions_cache.raw_completions = self.ComputeCandidatesInner(
request_data )
self._completions_cache.line = request_data[ 'line_num' ]
self._completions_cache.column = request_data[ 'start_column' ]
return self._completions_cache.raw_completions
def ComputeCandidatesInner( self, request_data ):
pass
def DefinedSubcommands( self ): def DefinedSubcommands( self ):
@ -224,46 +209,6 @@ class Completer( object ):
return matches return matches
def CandidatesForQueryAsyncInner( self, query, start_column ):
pass
# It's highly likely you DON'T want to override this function but the *Inner
# version of it.
def AsyncCandidateRequestReady( self ):
if self.completions_cache:
return True
else:
return self.AsyncCandidateRequestReadyInner()
def AsyncCandidateRequestReadyInner( self ):
if not self.completions_future:
# We return True so that the caller can extract the default value from the
# future
return True
return self.completions_future.ResultsReady()
# It's highly likely you DON'T want to override this function but the *Inner
# version of it.
def CandidatesFromStoredRequest( self ):
if self.completions_cache:
return self.completions_cache.filtered_completions
else:
self.completions_cache = CompletionsCache()
self.completions_cache.raw_completions = self.CandidatesFromStoredRequestInner()
self.completions_cache.line = self.request_data[ 'line_num' ]
self.completions_cache.column = self.request_data[ 'start_column' ]
return self.completions_cache.raw_completions
def CandidatesFromStoredRequestInner( self ):
if not self.completions_future:
return []
return self.completions_future.GetResults()
def OnFileReadyToParse( self, request_data ): def OnFileReadyToParse( self, request_data ):
pass pass

116
python/ycm/completers/cpp/clang_completer.py
View File

@ -30,8 +30,8 @@ CLANG_FILETYPES = set( [ 'c', 'cpp', 'objc', 'objcpp' ] )
MIN_LINES_IN_FILE_TO_PARSE = 5 MIN_LINES_IN_FILE_TO_PARSE = 5
PARSING_FILE_MESSAGE = 'Still parsing file, no completions yet.' PARSING_FILE_MESSAGE = 'Still parsing file, no completions yet.'
NO_COMPILE_FLAGS_MESSAGE = 'Still no compile flags, no completions yet.' NO_COMPILE_FLAGS_MESSAGE = 'Still no compile flags, no completions yet.'
NO_COMPLETIONS_MESSAGE = 'No completions found; errors in the file?'
INVALID_FILE_MESSAGE = 'File is invalid.' INVALID_FILE_MESSAGE = 'File is invalid.'
NO_COMPLETIONS_MESSAGE = 'No completions found; errors in the file?'
FILE_TOO_SHORT_MESSAGE = ( FILE_TOO_SHORT_MESSAGE = (
'File is less than {} lines long; not compiling.'.format( 'File is less than {} lines long; not compiling.'.format(
MIN_LINES_IN_FILE_TO_PARSE ) ) MIN_LINES_IN_FILE_TO_PARSE ) )
@ -41,25 +41,14 @@ NO_DIAGNOSTIC_MESSAGE = 'No diagnostic for current line!'
class ClangCompleter( Completer ): class ClangCompleter( Completer ):
def __init__( self, user_options ): def __init__( self, user_options ):
super( ClangCompleter, self ).__init__( user_options ) super( ClangCompleter, self ).__init__( user_options )
self.max_diagnostics_to_display = user_options[ self._max_diagnostics_to_display = user_options[
'max_diagnostics_to_display' ] 'max_diagnostics_to_display' ]
self.completer = ycm_core.ClangCompleter() self._completer = ycm_core.ClangCompleter()
self.completer.EnableThreading() self._last_prepared_diagnostics = []
self.last_prepared_diagnostics = [] self._flags = Flags()
self.parse_future = None self._diagnostic_store = None
self.flags = Flags()
self.diagnostic_store = None
self._logger = logging.getLogger( __name__ ) self._logger = logging.getLogger( __name__ )
# We set this flag when a compilation request comes in while one is already
# in progress. We use this to trigger the pending request after the previous
# one completes (from GetDiagnosticsForCurrentFile because that's the only
# method that knows when the compilation has finished).
# TODO: Remove this now that we have multiple threads in the server; the
# subsequent requests that want to parse will just block until the current
# parse is done and will then proceed.
self.extra_parse_desired = False
def SupportedFiletypes( self ): def SupportedFiletypes( self ):
return CLANG_FILETYPES return CLANG_FILETYPES
@ -84,53 +73,35 @@ class ClangCompleter( Completer ):
return files return files
def CandidatesForQueryAsync( self, request_data ): def ComputeCandidatesInner( self, request_data ):
filename = request_data[ 'filepath' ] filename = request_data[ 'filepath' ]
if not filename: if not filename:
return return
if self.completer.UpdatingTranslationUnit( ToUtf8IfNeeded( filename ) ): if self._completer.UpdatingTranslationUnit( ToUtf8IfNeeded( filename ) ):
self.completions_future = None
self._logger.info( PARSING_FILE_MESSAGE ) self._logger.info( PARSING_FILE_MESSAGE )
return responses.BuildDisplayMessageResponse( raise RuntimeError( PARSING_FILE_MESSAGE )
PARSING_FILE_MESSAGE )
flags = self._FlagsForRequest( request_data ) flags = self._FlagsForRequest( request_data )
if not flags: if not flags:
self.completions_future = None
self._logger.info( NO_COMPILE_FLAGS_MESSAGE ) self._logger.info( NO_COMPILE_FLAGS_MESSAGE )
return responses.BuildDisplayMessageResponse( raise RuntimeError( NO_COMPILE_FLAGS_MESSAGE )
NO_COMPILE_FLAGS_MESSAGE )
# TODO: sanitize query, probably in C++ code
files = ycm_core.UnsavedFileVec()
query = request_data[ 'query' ]
if not query:
files = self.GetUnsavedFilesVector( request_data ) files = self.GetUnsavedFilesVector( request_data )
line = request_data[ 'line_num' ] + 1 line = request_data[ 'line_num' ] + 1
column = request_data[ 'start_column' ] + 1 column = request_data[ 'start_column' ] + 1
self.completions_future = ( results = self._completer.CandidatesForLocationInFile(
self.completer.CandidatesForQueryAndLocationInFileAsync(
ToUtf8IfNeeded( query ),
ToUtf8IfNeeded( filename ), ToUtf8IfNeeded( filename ),
line, line,
column, column,
files, files,
flags ) ) flags )
def CandidatesFromStoredRequest( self ):
if not self.completions_future:
return []
results = [ ConvertCompletionData( x ) for x in
self.completions_future.GetResults() ]
if not results: if not results:
self._logger.warning( NO_COMPLETIONS_MESSAGE ) self._logger.warning( NO_COMPLETIONS_MESSAGE )
raise RuntimeError( NO_COMPLETIONS_MESSAGE ) raise RuntimeError( NO_COMPLETIONS_MESSAGE )
return results
return [ ConvertCompletionData( x ) for x in results ]
def DefinedSubcommands( self ): def DefinedSubcommands( self ):
@ -170,7 +141,7 @@ class ClangCompleter( Completer ):
files = self.GetUnsavedFilesVector( request_data ) files = self.GetUnsavedFilesVector( request_data )
line = request_data[ 'line_num' ] + 1 line = request_data[ 'line_num' ] + 1
column = request_data[ 'column_num' ] + 1 column = request_data[ 'column_num' ] + 1
return getattr( self.completer, goto_function )( return getattr( self._completer, goto_function )(
ToUtf8IfNeeded( filename ), ToUtf8IfNeeded( filename ),
line, line,
column, column,
@ -212,73 +183,63 @@ class ClangCompleter( Completer ):
def _ClearCompilationFlagCache( self ): def _ClearCompilationFlagCache( self ):
self.flags.Clear() self._flags.Clear()
def OnFileReadyToParse( self, request_data ): def OnFileReadyToParse( self, request_data ):
filename = request_data[ 'filepath' ] filename = request_data[ 'filepath' ]
contents = request_data[ 'file_data' ][ filename ][ 'contents' ] contents = request_data[ 'file_data' ][ filename ][ 'contents' ]
if contents.count( '\n' ) < MIN_LINES_IN_FILE_TO_PARSE: if contents.count( '\n' ) < MIN_LINES_IN_FILE_TO_PARSE:
self.parse_future = None
self._logger.warning( FILE_TOO_SHORT_MESSAGE ) self._logger.warning( FILE_TOO_SHORT_MESSAGE )
raise ValueError( FILE_TOO_SHORT_MESSAGE ) raise ValueError( FILE_TOO_SHORT_MESSAGE )
if not filename: if not filename:
self._logger.warning( INVALID_FILE_MESSAGE ) self._logger.warning( INVALID_FILE_MESSAGE )
return responses.BuildDisplayMessageResponse( raise ValueError( INVALID_FILE_MESSAGE )
INVALID_FILE_MESSAGE )
if self.completer.UpdatingTranslationUnit( ToUtf8IfNeeded( filename ) ):
self.extra_parse_desired = True
return
flags = self._FlagsForRequest( request_data ) flags = self._FlagsForRequest( request_data )
if not flags: if not flags:
self.parse_future = None
self._logger.info( NO_COMPILE_FLAGS_MESSAGE ) self._logger.info( NO_COMPILE_FLAGS_MESSAGE )
return responses.BuildDisplayMessageResponse( raise ValueError( NO_COMPILE_FLAGS_MESSAGE )
NO_COMPILE_FLAGS_MESSAGE )
self.parse_future = self.completer.UpdateTranslationUnitAsync( self._completer.UpdateTranslationUnit(
ToUtf8IfNeeded( filename ), ToUtf8IfNeeded( filename ),
self.GetUnsavedFilesVector( request_data ), self.GetUnsavedFilesVector( request_data ),
flags ) flags )
self.extra_parse_desired = False
def OnBufferUnload( self, request_data ): def OnBufferUnload( self, request_data ):
self.completer.DeleteCachesForFileAsync( self._completer.DeleteCachesForFile(
ToUtf8IfNeeded( request_data[ 'unloaded_buffer' ] ) ) ToUtf8IfNeeded( request_data[ 'unloaded_buffer' ] ) )
def DiagnosticsForCurrentFileReady( self ): def DiagnosticsForCurrentFileReady( self ):
if not self.parse_future: # if not self.parse_future:
return False # return False
# return self.parse_future.ResultsReady()
return self.parse_future.ResultsReady() pass
def GettingCompletions( self, request_data ): def GettingCompletions( self, request_data ):
return self.completer.UpdatingTranslationUnit( return self._completer.UpdatingTranslationUnit(
ToUtf8IfNeeded( request_data[ 'filepath' ] ) ) ToUtf8IfNeeded( request_data[ 'filepath' ] ) )
def GetDiagnosticsForCurrentFile( self, request_data ): def GetDiagnosticsForCurrentFile( self, request_data ):
filename = request_data[ 'filepath' ] filename = request_data[ 'filepath' ]
if self.DiagnosticsForCurrentFileReady(): if self.DiagnosticsForCurrentFileReady():
diagnostics = self.completer.DiagnosticsForFile( diagnostics = self._completer.DiagnosticsForFile(
ToUtf8IfNeeded( filename ) ) ToUtf8IfNeeded( filename ) )
self.diagnostic_store = DiagnosticsToDiagStructure( diagnostics ) self._diagnostic_store = DiagnosticsToDiagStructure( diagnostics )
self.last_prepared_diagnostics = [ self._last_prepared_diagnostics = [
responses.BuildDiagnosticData( x ) for x in responses.BuildDiagnosticData( x ) for x in
diagnostics[ : self.max_diagnostics_to_display ] ] diagnostics[ : self._max_diagnostics_to_display ] ]
self.parse_future = None # self.parse_future = None
if self.extra_parse_desired: # if self.extra_parse_desired:
self.OnFileReadyToParse( request_data ) # self.OnFileReadyToParse( request_data )
return self.last_prepared_diagnostics return self._last_prepared_diagnostics
def GetDetailedDiagnostic( self, request_data ): def GetDetailedDiagnostic( self, request_data ):
@ -286,11 +247,11 @@ class ClangCompleter( Completer ):
current_column = request_data[ 'column_num' ] + 1 current_column = request_data[ 'column_num' ] + 1
current_file = request_data[ 'filepath' ] current_file = request_data[ 'filepath' ]
if not self.diagnostic_store: if not self._diagnostic_store:
return responses.BuildDisplayMessageResponse( return responses.BuildDisplayMessageResponse(
NO_DIAGNOSTIC_MESSAGE ) NO_DIAGNOSTIC_MESSAGE )
diagnostics = self.diagnostic_store[ current_file ][ current_line ] diagnostics = self._diagnostic_store[ current_file ][ current_line ]
if not diagnostics: if not diagnostics:
return responses.BuildDisplayMessageResponse( return responses.BuildDisplayMessageResponse(
NO_DIAGNOSTIC_MESSAGE ) NO_DIAGNOSTIC_MESSAGE )
@ -308,11 +269,6 @@ class ClangCompleter( Completer ):
closest_diagnostic.long_formatted_text_ ) closest_diagnostic.long_formatted_text_ )
def ShouldUseNow( self, request_data ):
# We don't want to use the Completer API cache, we use one in the C++ code.
return self.ShouldUseNowInner( request_data )
def DebugInfo( self, request_data ): def DebugInfo( self, request_data ):
filename = request_data[ 'filepath' ] filename = request_data[ 'filepath' ]
if not filename: if not filename:
@ -328,7 +284,7 @@ class ClangCompleter( Completer ):
if 'compilation_flags' in request_data: if 'compilation_flags' in request_data:
return PrepareFlagsForClang( request_data[ 'compilation_flags' ], return PrepareFlagsForClang( request_data[ 'compilation_flags' ],
filename ) filename )
return self.flags.FlagsForFile( filename ) return self._flags.FlagsForFile( filename )
# TODO: Make this work again # TODO: Make this work again
# def DiagnosticToDict( diagnostic ): # def DiagnosticToDict( diagnostic ):

6
python/ycm/completers/cs/cs_completer.py
View File

@ -21,7 +21,7 @@
import os import os
from sys import platform from sys import platform
import glob import glob
from ycm.completers.threaded_completer import ThreadedCompleter from ycm.completers.completer import Completer
from ycm.server import responses from ycm.server import responses
from ycm import utils from ycm import utils
import urllib2 import urllib2
@ -36,7 +36,7 @@ SERVER_NOT_FOUND_MSG = ( 'OmniSharp server binary not found at {0}. ' +
'Did you compile it? You can do so by running ' + 'Did you compile it? You can do so by running ' +
'"./install.sh --omnisharp-completer".' ) '"./install.sh --omnisharp-completer".' )
class CsharpCompleter( ThreadedCompleter ): class CsharpCompleter( Completer ):
""" """
A Completer that uses the Omnisharp server as completion engine. A Completer that uses the Omnisharp server as completion engine.
""" """
@ -61,7 +61,7 @@ class CsharpCompleter( ThreadedCompleter ):
return [ 'cs' ] return [ 'cs' ]
def ComputeCandidates( self, request_data ): def ComputeCandidatesInner( self, request_data ):
return [ responses.BuildCompletionData( return [ responses.BuildCompletionData(
completion[ 'CompletionText' ], completion[ 'CompletionText' ],
completion[ 'DisplayText' ], completion[ 'DisplayText' ],

6
python/ycm/completers/general/filename_completer.py
View File

@ -19,12 +19,12 @@
import os import os
import re import re
from ycm.completers.threaded_completer import ThreadedCompleter from ycm.completers.completer import Completer
from ycm.completers.cpp.clang_completer import InCFamilyFile from ycm.completers.cpp.clang_completer import InCFamilyFile
from ycm.completers.cpp.flags import Flags from ycm.completers.cpp.flags import Flags
from ycm.server import responses from ycm.server import responses
class FilenameCompleter( ThreadedCompleter ): class FilenameCompleter( Completer ):
""" """
General completer that provides filename and filepath completions. General completer that provides filename and filepath completions.
""" """
@ -73,7 +73,7 @@ class FilenameCompleter( ThreadedCompleter ):
return [] return []
def ComputeCandidates( self, request_data ): def ComputeCandidatesInner( self, request_data ):
current_line = request_data[ 'line_value' ] current_line = request_data[ 'line_value' ]
start_column = request_data[ 'start_column' ] start_column = request_data[ 'start_column' ]
filepath = request_data[ 'filepath' ] filepath = request_data[ 'filepath' ]

15
python/ycm/completers/general/general_completer_store.py
View File

@ -74,20 +74,13 @@ class GeneralCompleterStore( Completer ):
return should_use_now return should_use_now
def CandidatesForQueryAsync( self, request_data ): def ComputeCandidates( self, request_data ):
for completer in self._current_query_completers: if not self.ShouldUseNow( request_data ):
completer.CandidatesForQueryAsync( request_data ) return []
def AsyncCandidateRequestReady( self ):
return all( x.AsyncCandidateRequestReady() for x in
self._current_query_completers )
def CandidatesFromStoredRequest( self ):
candidates = [] candidates = []
for completer in self._current_query_completers: for completer in self._current_query_completers:
candidates += completer.CandidatesFromStoredRequest() candidates += completer.ComputeCandidates( request_data )
return candidates return candidates

16
python/ycm/completers/general/ultisnips_completer.py
View File

@ -33,23 +33,17 @@ class UltiSnipsCompleter( GeneralCompleter ):
self._filtered_candidates = None self._filtered_candidates = None
def ShouldUseNowInner( self, request_data ): def ShouldUseNow( self, request_data ):
return self.QueryLengthAboveMinThreshold( request_data ) return self.QueryLengthAboveMinThreshold( request_data )
def CandidatesForQueryAsync( self, request_data ): def ComputeCandidates( self, request_data ):
self._filtered_candidates = self.FilterAndSortCandidates( if not self.ShouldUseNow( request_data ):
return []
return self.FilterAndSortCandidates(
self._candidates, request_data[ 'query' ] ) self._candidates, request_data[ 'query' ] )
def AsyncCandidateRequestReady( self ):
return True
def CandidatesFromStoredRequest( self ):
return self._filtered_candidates if self._filtered_candidates else []
def OnBufferVisit( self, request_data ): def OnBufferVisit( self, request_data ):
raw_candidates = request_data[ 'ultisnips_snippets' ] raw_candidates = request_data[ 'ultisnips_snippets' ]
self._candidates = [ self._candidates = [

6
python/ycm/completers/python/jedi_completer.py
View File

@ -19,7 +19,7 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with YouCompleteMe. If not, see <http://www.gnu.org/licenses/>. # along with YouCompleteMe. If not, see <http://www.gnu.org/licenses/>.
from ycm.completers.threaded_completer import ThreadedCompleter from ycm.completers.completer import Completer
from ycm.server import responses from ycm.server import responses
import sys import sys
@ -38,7 +38,7 @@ except ImportError:
sys.path.pop( 0 ) sys.path.pop( 0 )
class JediCompleter( ThreadedCompleter ): class JediCompleter( Completer ):
""" """
A Completer that uses the Jedi completion engine. A Completer that uses the Jedi completion engine.
https://jedi.readthedocs.org/en/latest/ https://jedi.readthedocs.org/en/latest/
@ -63,7 +63,7 @@ class JediCompleter( ThreadedCompleter ):
return jedi.Script( contents, line, column, filename ) return jedi.Script( contents, line, column, filename )
def ComputeCandidates( self, request_data ): def ComputeCandidatesInner( self, request_data ):
script = self._GetJediScript( request_data ) script = self._GetJediScript( request_data )
return [ responses.BuildCompletionData( return [ responses.BuildCompletionData(
str( completion.name ), str( completion.name ),

100
python/ycm/completers/threaded_completer.py
View File

@ -1,100 +0,0 @@
#!/usr/bin/env python
#
# Copyright (C) 2011, 2012 Strahinja Val Markovic <val@markovic.io>
#
# This file is part of YouCompleteMe.
#
# YouCompleteMe is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# YouCompleteMe is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with YouCompleteMe. If not, see <http://www.gnu.org/licenses/>.
import abc
from threading import Thread, Event
from ycm.completers.completer import Completer
class ThreadedCompleter( Completer ):
"""A subclass of Completer that abstracts away the use of a background thread.
This is a great class to subclass for your custom completer. It will provide
you with async computation of candidates when you implement the
ComputeCandidates() method; no need to worry about threads, locks, events or
similar.
If you use this class as the base class for your Completer, you DON'T need to
provide implementations for the CandidatesForQueryAsync(),
AsyncCandidateRequestReady() and CandidatesFromStoredRequest() functions. Just
implement ComputeCandidates().
For examples of subclasses of this class, see the following files:
python/completers/general/filename_completer.py
"""
def __init__( self, user_options ):
super( ThreadedCompleter, self ).__init__( user_options )
self._query_ready = Event()
self._candidates_ready = Event()
self._candidates = None
self._start_completion_thread()
def _start_completion_thread( self ):
self._completion_thread = Thread( target=self.SetCandidates )
self._completion_thread.daemon = True
self._completion_thread.start()
def CandidatesForQueryAsyncInner( self, request_data ):
self._candidates = None
self._candidates_ready.clear()
self._request_data = request_data
self._query_ready.set()
def AsyncCandidateRequestReadyInner( self ):
return WaitAndClearIfSet( self._candidates_ready, timeout=0.005 )
def CandidatesFromStoredRequestInner( self ):
return self._candidates or []
@abc.abstractmethod
def ComputeCandidates( self, request_data ):
"""This function should compute the candidates to show to the user.
The return value should be of the same type as that for
CandidatesFromStoredRequest()."""
pass
def SetCandidates( self ):
while True:
try:
WaitAndClearIfSet( self._query_ready )
self._candidates = self.ComputeCandidates( self._request_data )
except:
self._query_ready.clear()
self._candidates = []
self._candidates_ready.set()
def WaitAndClearIfSet( event, timeout=None ):
"""Given an |event| and a |timeout|, waits for the event a maximum of timeout
seconds. After waiting, clears the event if it's set and returns the state of
the event before it was cleared."""
# We can't just do flag_is_set = event.wait( timeout ) because that breaks on
# Python 2.6
event.wait( timeout )
flag_is_set = event.is_set()
if flag_is_set:
event.clear()
return flag_is_set

48
python/ycm/server/tests/basic_test.py
View File

@ -21,6 +21,7 @@ from webtest import TestApp
from .. import ycmd from .. import ycmd
from ..responses import BuildCompletionData from ..responses import BuildCompletionData
from nose.tools import ok_, eq_, with_setup from nose.tools import ok_, eq_, with_setup
from hamcrest import assert_that, has_items, has_entry
import bottle import bottle
bottle.debug( True ) bottle.debug( True )
@ -41,6 +42,11 @@ def RequestDataForFileWithContents( filename, contents = None ):
} }
# TODO: Make the other tests use this helper too instead of BuildCompletionData
def CompletionEntryMatcher( insertion_text ):
return has_entry( 'insertion_text', insertion_text )
def Setup(): def Setup():
ycmd.SetServerStateToDefaults() ycmd.SetServerStateToDefaults()
@ -69,6 +75,48 @@ def GetCompletions_IdentifierCompleter_Works_test():
app.post_json( '/get_completions', completion_data ).json ) app.post_json( '/get_completions', completion_data ).json )
@with_setup( Setup )
def GetCompletions_ClangCompleter_Works_test():
app = TestApp( ycmd.app )
contents = """
struct Foo {
int x;
int y;
char c;
};
int main()
{
Foo foo;
foo.
}
"""
filename = '/foo.cpp'
completion_data = {
'compilation_flags': ['-x', 'c++'],
# 0-based line and column!
'query': '',
'line_num': 10,
'column_num': 6,
'start_column': 6,
'line_value': ' foo.',
'filetypes': ['cpp'],
'filepath': filename,
'file_data': {
filename: {
'contents': contents,
'filetypes': ['cpp']
}
}
}
results = app.post_json( '/get_completions', completion_data ).json
assert_that( results, has_items( CompletionEntryMatcher( 'c' ),
CompletionEntryMatcher( 'x' ),
CompletionEntryMatcher( 'y' ) ) )
@with_setup( Setup ) @with_setup( Setup )
def GetCompletions_IdentifierCompleter_SyntaxKeywordsAdded_test(): def GetCompletions_IdentifierCompleter_SyntaxKeywordsAdded_test():
app = TestApp( ycmd.app ) app = TestApp( ycmd.app )

12
python/ycm/server/ycmd.py
View File

@ -31,7 +31,6 @@ sys.path.insert( 0, os.path.join(
'../..' ) ) '../..' ) )
import logging import logging
import time
import json import json
import bottle import bottle
from bottle import run, request, response from bottle import run, request, response
@ -89,16 +88,7 @@ def GetCompletions():
do_filetype_completion else do_filetype_completion else
SERVER_STATE.GetGeneralCompleter() ) SERVER_STATE.GetGeneralCompleter() )
# This is necessary so that general_completer_store fills up return _JsonResponse( completer.ComputeCandidates( request_data ) )
# _current_query_completers.
# TODO: Fix this.
completer.ShouldUseNow( request_data )
# TODO: This should not be async anymore, server is multi-threaded
completer.CandidatesForQueryAsync( request_data )
while not completer.AsyncCandidateRequestReady():
time.sleep( 0.03 )
return _JsonResponse( completer.CandidatesFromStoredRequest() )
@app.get( '/user_options' ) @app.get( '/user_options' )