vim/YouCompleteMe
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

No more threading in completers!

Browse Source 
The server is multi-threaded and will spawn a new thread for each new request.
Thus, the completers need not manage their own threads or even provide async
APIs; we _want_ them to block because now were implementing the request-response
networking API.

The client gets the async API through the network (i.e., it can do something
else while the request is pending).
This commit is contained in:
Strahinja Val Markovic 2013-09-30 13:40:25 -07:00
parent e38d145a47
commit 6c53bad58f
21 changed files with 189 additions and 1136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

343
cpp/ycm/ClangCompleter/ClangCompleter.cpp
View File

@ -1,4 +1,4 @@
// Copyright (C) 2011, 2012 Strahinja Val Markovic <val@markovic.io>
// Copyright (C) 2011-2013 Strahinja Val Markovic <val@markovic.io>
//
// This file is part of YouCompleteMe.
//
@ -28,43 +28,17 @@
#include "ClangUtils.h"
#include <clang-c/Index.h>
#include <boost/make_shared.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/cxx11/any_of.hpp>
#include <boost/shared_ptr.hpp>
using boost::algorithm::any_of;
using boost::algorithm::is_upper;
using boost::bind;
using boost::cref;
using boost::function;
using boost::lock_guard;
using boost::make_shared;
using boost::mutex;
using boost::packaged_task;
using boost::ref;
using boost::shared_lock;
using boost::shared_mutex;
using boost::shared_ptr;
using boost::thread;
using boost::thread_interrupted;
using boost::try_to_lock_t;
using boost::unique_future;
using boost::unique_lock;
using boost::unordered_map;
namespace YouCompleteMe {
extern const unsigned int MAX_ASYNC_THREADS;
extern const unsigned int MIN_ASYNC_THREADS;
ClangCompleter::ClangCompleter()
: clang_index_( clang_createIndex( 0, 0 ) ),
translation_unit_store_( clang_index_ ),
candidate_repository_( CandidateRepository::Instance() ),
threading_enabled_( false ),
time_to_die_( false ),
clang_data_ready_( false ) {
translation_unit_store_( clang_index_ ) {
// The libclang docs don't say what is the default value for crash recovery.
// I'm pretty sure it's turned on by default, but I'm not going to take any
// chances.
@ -73,35 +47,18 @@ ClangCompleter::ClangCompleter()
ClangCompleter::~ClangCompleter() {
{
unique_lock< shared_mutex > lock( time_to_die_mutex_ );
time_to_die_ = true;
}
sorting_threads_.interrupt_all();
sorting_threads_.join_all();
if ( clang_thread_ ) {
clang_thread_->interrupt();
clang_thread_->join();
}
// We need to destroy all TUs before calling clang_disposeIndex because
// the translation units need to be destroyed before the index is destroyed.
// Technically, a thread could still be holding onto a shared_ptr<TU> object
// when we destroy the clang index, but since we're shutting down, we don't
// really care.
// In practice, this situation shouldn't happen because the server threads are
// Python deamon threads and will all be killed before the main thread exits.
translation_unit_store_.RemoveAll();
clang_disposeIndex( clang_index_ );
}
// We need this mostly so that we can not use it in tests. Apparently the
// GoogleTest framework goes apeshit on us (on some platforms, in some
// occasions) if we enable threads by default.
void ClangCompleter::EnableThreading() {
threading_enabled_ = true;
InitThreads();
}
std::vector< Diagnostic > ClangCompleter::DiagnosticsForFile(
std::string filename ) {
shared_ptr< TranslationUnit > unit = translation_unit_store_.Get( filename );
@ -156,30 +113,6 @@ void ClangCompleter::UpdateTranslationUnit(
}
Future< void > ClangCompleter::UpdateTranslationUnitAsync(
std::string filename,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags ) {
function< void() > functor =
boost::bind( &ClangCompleter::UpdateTranslationUnit,
boost::ref( *this ),
boost::move( filename ),
boost::move( unsaved_files ),
boost::move( flags ) );
shared_ptr< ClangPackagedTask > clang_packaged_task =
make_shared< ClangPackagedTask >();
clang_packaged_task->parsing_task_ = packaged_task< void >(
boost::move( functor ) );
unique_future< void > future =
clang_packaged_task->parsing_task_.get_future();
clang_task_.Set( clang_packaged_task );
return Future< void >( boost::move( future ) );
}
std::vector< CompletionData >
ClangCompleter::CandidatesForLocationInFile(
std::string filename,
@ -199,56 +132,6 @@ ClangCompleter::CandidatesForLocationInFile(
}
Future< AsyncCompletions >
ClangCompleter::CandidatesForQueryAndLocationInFileAsync(
std::string query,
std::string filename,
int line,
int column,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags ) {
// TODO: throw exception when threading is not enabled and this is called
if ( !threading_enabled_ )
return Future< AsyncCompletions >();
bool skip_clang_result_cache = ShouldSkipClangResultCache( query,
line,
column );
if ( skip_clang_result_cache ) {
// The clang thread is busy, return nothing
if ( UpdatingTranslationUnit( filename ) )
return Future< AsyncCompletions >();
{
lock_guard< mutex > lock( clang_data_ready_mutex_ );
clang_data_ready_ = false;
}
// Needed to "reset" the sorting threads to the start of their loop. This
// way any threads blocking on a read in sorting_task_.Get() are reset to
// wait on the clang_data_ready_condition_variable_.
sorting_threads_.interrupt_all();
}
// the sorting task needs to be set before the clang task (if any) just in
// case the clang task finishes (and therefore notifies a sorting thread to
// consume a sorting task) before the sorting task is set
unique_future< AsyncCompletions > future;
CreateSortingTask( query, future );
if ( skip_clang_result_cache ) {
CreateClangTask( boost::move( filename ),
line,
column,
boost::move( unsaved_files ),
boost::move( flags ) );
}
return Future< AsyncCompletions >( boost::move( future ) );
}
Location ClangCompleter::GetDeclarationLocation(
std::string filename,
int line,
@ -283,216 +166,8 @@ Location ClangCompleter::GetDefinitionLocation(
}
void ClangCompleter::DeleteCachesForFileAsync( std::string filename ) {
file_cache_delete_stack_.Push( filename );
}
void ClangCompleter::DeleteCaches() {
std::vector< std::string > filenames;
if ( !file_cache_delete_stack_.PopAllNoWait( filenames ) )
return;
foreach( const std::string & filename, filenames ) {
void ClangCompleter::DeleteCachesForFile( std::string filename ) {
translation_unit_store_.Remove( filename );
}
}
bool ClangCompleter::ShouldSkipClangResultCache( const std::string &query,
int line,
int column ) {
// We need query.empty() in addition to the second check because if we don't
// have it, then we have a problem in the following situation:
// The user has a variable 'foo' of type 'A' and a variable 'bar' of type 'B'.
// He then types in 'foo.' and we store the clang results and also return
// them. The user then deletes that code and types in 'bar.'. Without the
// query.empty() check we would return the results from the cache here (it's
// the same line and column!) which would be incorrect.
return query.empty() || latest_clang_results_
.NewPositionDifferentFromStoredPosition( line, column );
}
// Copy-ctor for unique_future is private in C++03 mode so we need to take it as
// an out param
void ClangCompleter::CreateSortingTask(
const std::string &query,
unique_future< AsyncCompletions > &future ) {
// Careful! The code in this function may burn your eyes.
function< CompletionDatas( const CompletionDatas & ) >
sort_candidates_for_query_functor =
boost::bind( &ClangCompleter::SortCandidatesForQuery,
boost::ref( *this ),
query,
_1 );
function< CompletionDatas() > operate_on_completion_data_functor =
boost::bind( &ClangResultsCache::OperateOnCompletionDatas< CompletionDatas >,
boost::cref( latest_clang_results_ ),
boost::move( sort_candidates_for_query_functor ) );
shared_ptr< packaged_task< AsyncCompletions > > task =
boost::make_shared< packaged_task< AsyncCompletions > >(
boost::bind( ReturnValueAsShared< std::vector< CompletionData > >,
boost::move( operate_on_completion_data_functor ) ) );
future = task->get_future();
sorting_task_.Set( task );
}
void ClangCompleter::CreateClangTask(
std::string filename,
int line,
int column,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags ) {
latest_clang_results_.ResetWithNewLineAndColumn( line, column );
function< CompletionDatas() > candidates_for_location_functor =
boost::bind( &ClangCompleter::CandidatesForLocationInFile,
boost::ref( *this ),
boost::move( filename ),
line,
column,
boost::move( unsaved_files ),
boost::move( flags ) );
shared_ptr< ClangPackagedTask > clang_packaged_task =
make_shared< ClangPackagedTask >();
clang_packaged_task->completions_task_ =
packaged_task< AsyncCompletions >(
boost::bind( ReturnValueAsShared< std::vector< CompletionData > >,
boost::move( candidates_for_location_functor ) ) );
clang_task_.Set( clang_packaged_task );
}
std::vector< CompletionData > ClangCompleter::SortCandidatesForQuery(
const std::string &query,
const std::vector< CompletionData > &completion_datas ) {
Bitset query_bitset = LetterBitsetFromString( query );
bool query_has_uppercase_letters = any_of( query, is_upper() );
std::vector< const Candidate * > repository_candidates =
candidate_repository_.GetCandidatesForStrings( completion_datas );
std::vector< ResultAnd< CompletionData * > > data_and_results;
for ( uint i = 0; i < repository_candidates.size(); ++i ) {
const Candidate *candidate = repository_candidates[ i ];
if ( !candidate->MatchesQueryBitset( query_bitset ) )
continue;
Result result = candidate->QueryMatchResult( query,
query_has_uppercase_letters );
if ( result.IsSubsequence() ) {
ResultAnd< CompletionData * > data_and_result( &completion_datas[ i ],
result );
data_and_results.push_back( boost::move( data_and_result ) );
}
}
std::sort( data_and_results.begin(), data_and_results.end() );
std::vector< CompletionData > sorted_completion_datas;
sorted_completion_datas.reserve( data_and_results.size() );
foreach ( const ResultAnd< CompletionData * > &data_and_result,
data_and_results ) {
sorted_completion_datas.push_back( *data_and_result.extra_object_ );
}
return sorted_completion_datas;
}
void ClangCompleter::InitThreads() {
int threads_to_create =
std::max( MIN_ASYNC_THREADS,
std::min( MAX_ASYNC_THREADS, thread::hardware_concurrency() ) );
for ( int i = 0; i < threads_to_create; ++i ) {
sorting_threads_.create_thread( bind( &ClangCompleter::SortingThreadMain,
boost::ref( *this ) ) );
}
clang_thread_.reset( new thread( &ClangCompleter::ClangThreadMain,
boost::ref( *this ) ) );
}
void ClangCompleter::ClangThreadMain() {
while ( true ) {
try {
shared_ptr< ClangPackagedTask > task = clang_task_.Get();
bool has_completions_task = task->completions_task_.valid();
if ( has_completions_task )
task->completions_task_();
else
task->parsing_task_();
if ( !has_completions_task ) {
DeleteCaches();
continue;
}
unique_future< AsyncCompletions > future =
task->completions_task_.get_future();
latest_clang_results_.SetCompletionDatas( *future.get() );
{
lock_guard< mutex > lock( clang_data_ready_mutex_ );
clang_data_ready_ = true;
}
clang_data_ready_condition_variable_.notify_all();
} catch ( thread_interrupted & ) {
shared_lock< shared_mutex > lock( time_to_die_mutex_ );
if ( time_to_die_ )
return;
// else do nothing and re-enter the loop
}
}
}
void ClangCompleter::SortingThreadMain() {
while ( true ) {
try {
{
unique_lock< mutex > lock( clang_data_ready_mutex_ );
while ( !clang_data_ready_ ) {
clang_data_ready_condition_variable_.wait( lock );
}
}
shared_ptr< packaged_task< AsyncCompletions > > task =
sorting_task_.Get();
( *task )();
} catch ( thread_interrupted & ) {
shared_lock< shared_mutex > lock( time_to_die_mutex_ );
if ( time_to_die_ )
return;
// else do nothing and re-enter the loop
}
}
}

106
cpp/ycm/ClangCompleter/ClangCompleter.h
View File

@ -23,13 +23,9 @@
#include "Future.h"
#include "UnsavedFile.h"
#include "Diagnostic.h"
#include "ClangResultsCache.h"
#include "TranslationUnitStore.h"
#include <boost/utility.hpp>
#include <boost/thread/future.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/unordered_map.hpp>
#include <string>
@ -37,19 +33,12 @@ typedef struct CXTranslationUnitImpl *CXTranslationUnit;
namespace YouCompleteMe {
class CandidateRepository;
class TranslationUnit;
struct CompletionData;
struct Location;
typedef std::vector< CompletionData > CompletionDatas;
typedef boost::shared_ptr< CompletionDatas > AsyncCompletions;
typedef boost::unordered_map < std::string,
boost::shared_ptr <
std::vector< std::string > > > FlagsForFile;
// TODO: document that all filename parameters must be absolute paths
class ClangCompleter : boost::noncopyable {
@ -57,8 +46,6 @@ public:
ClangCompleter();
~ClangCompleter();
void EnableThreading();
std::vector< Diagnostic > DiagnosticsForFile( std::string filename );
bool UpdatingTranslationUnit( const std::string &filename );
@ -68,20 +55,11 @@ public:
// need to ensure we own the memory.
// TODO: Change some of these params back to const ref where possible after we
// get the server up.
// TODO: Remove the async methods and the threads when the server is ready.
// Public because of unit tests (gtest is not very thread-friendly)
void UpdateTranslationUnit(
std::string filename,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags );
Future< void > UpdateTranslationUnitAsync(
std::string filename,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags );
// Public because of unit tests (gtest is not very thread-friendly)
std::vector< CompletionData > CandidatesForLocationInFile(
std::string filename,
int line,
@ -89,14 +67,6 @@ public:
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags );
Future< AsyncCompletions > CandidatesForQueryAndLocationInFileAsync(
std::string query,
std::string filename,
int line,
int column,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags );
Location GetDeclarationLocation(
std::string filename,
int line,
@ -111,55 +81,10 @@ public:
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags );
void DeleteCachesForFileAsync( std::string filename );
void DeleteCachesForFile( std::string filename );
private:
void DeleteCaches();
// This is basically a union. Only one of the two tasks is set to something
// valid, the other task is invalid. Which one is valid depends on the caller.
struct ClangPackagedTask {
boost::packaged_task< AsyncCompletions > completions_task_;
boost::packaged_task< void > parsing_task_;
};
typedef ConcurrentLatestValue <
boost::shared_ptr <
boost::packaged_task< AsyncCompletions > > > LatestSortingTask;
typedef ConcurrentLatestValue <
boost::shared_ptr< ClangPackagedTask > > LatestClangTask;
typedef ConcurrentStack< std::string > FileCacheDeleteStack;
bool ShouldSkipClangResultCache( const std::string &query,
int line,
int column );
void CreateSortingTask( const std::string &query,
boost::unique_future< AsyncCompletions > &future );
// NOTE: params are taken by value on purpose! With a C++11 compiler we can
// avoid internal copies if params are taken by value (move ctors FTW)
void CreateClangTask(
std::string filename,
int line,
int column,
std::vector< UnsavedFile > unsaved_files,
std::vector< std::string > flags );
std::vector< CompletionData > SortCandidatesForQuery(
const std::string &query,
const std::vector< CompletionData > &completion_datas );
void InitThreads();
void ClangThreadMain();
void SortingThreadMain();
/////////////////////////////
// PRIVATE MEMBER VARIABLES
/////////////////////////////
@ -167,35 +92,6 @@ private:
CXIndex clang_index_;
TranslationUnitStore translation_unit_store_;
CandidateRepository &candidate_repository_;
bool threading_enabled_;
// TODO: use boost.atomic for time_to_die_
bool time_to_die_;
boost::shared_mutex time_to_die_mutex_;
// TODO: use boost.atomic for clang_data_ready_
bool clang_data_ready_;
boost::mutex clang_data_ready_mutex_;
boost::condition_variable clang_data_ready_condition_variable_;
ClangResultsCache latest_clang_results_;
FileCacheDeleteStack file_cache_delete_stack_;
// Unfortunately clang is not thread-safe so we need to be careful when we
// access it. Only one thread at a time is allowed to access any single
// translation unit. Currently we only use one thread to access clang and that
// is the thread represented by clang_thread_.
boost::scoped_ptr< boost::thread > clang_thread_;
boost::thread_group sorting_threads_;
mutable LatestClangTask clang_task_;
mutable LatestSortingTask sorting_task_;
};
} // namespace YouCompleteMe

43
cpp/ycm/ClangCompleter/ClangResultsCache.cpp
View File

@ -1,43 +0,0 @@
// Copyright (C) 2011, 2012 Strahinja Val Markovic <val@markovic.io>
//
// This file is part of YouCompleteMe.
//
// YouCompleteMe is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// YouCompleteMe is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with YouCompleteMe. If not, see <http://www.gnu.org/licenses/>.
#include "ClangResultsCache.h"
#include "standard.h"
using boost::shared_mutex;
using boost::shared_lock;
using boost::unique_lock;
namespace YouCompleteMe {
bool ClangResultsCache::NewPositionDifferentFromStoredPosition( int new_line,
int new_colum )
const {
shared_lock< shared_mutex > reader_lock( access_mutex_ );
return line_ != new_line || column_ != new_colum;
}
void ClangResultsCache::ResetWithNewLineAndColumn( int new_line,
int new_colum ) {
unique_lock< shared_mutex > reader_lock( access_mutex_ );
line_ = new_line;
column_ = new_colum;
completion_datas_.clear();
}
} // namespace YouCompleteMe

84
cpp/ycm/ClangCompleter/ClangResultsCache.h
View File

@ -1,84 +0,0 @@
// Copyright (C) 2011, 2012 Strahinja Val Markovic <val@markovic.io>
//
// This file is part of YouCompleteMe.
//
// YouCompleteMe is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// YouCompleteMe is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with YouCompleteMe. If not, see <http://www.gnu.org/licenses/>.
#ifndef CLANGRESULTSCACHE_H_REUWM3RU
#define CLANGRESULTSCACHE_H_REUWM3RU
#include "CompletionData.h"
#include <vector>
#include <boost/thread/locks.hpp>
#include <boost/thread/shared_mutex.hpp>
#include <boost/function.hpp>
#include <boost/config.hpp>
#include <boost/utility.hpp>
namespace YouCompleteMe {
struct CompletionData;
class ClangResultsCache : boost::noncopyable {
public:
ClangResultsCache() : line_( -1 ), column_( -1 ) {}
bool NewPositionDifferentFromStoredPosition( int new_line, int new_colum )
const;
void ResetWithNewLineAndColumn( int new_line, int new_colum );
void SetCompletionDatas(
const std::vector< CompletionData > new_completions ) {
completion_datas_ = new_completions;
}
#ifndef BOOST_NO_RVALUE_REFERENCES
# ifdef __clang__
# pragma clang diagnostic push
# pragma clang diagnostic ignored "-Wc++98-compat"
# endif //#ifdef __clang__
void SetCompletionDatas( std::vector< CompletionData > &&new_completions ) {
completion_datas_ = new_completions;
}
# ifdef __clang__
# pragma clang diagnostic pop
# endif //#ifdef __clang__
#endif //#ifndef BOOST_NO_RVALUE_REFERENCES
template< typename T >
T OperateOnCompletionDatas(
boost::function< T( const std::vector< CompletionData >& ) > operation )
const {
boost::shared_lock< boost::shared_mutex > reader_lock( access_mutex_ );
return operation( completion_datas_ );
}
private:
int line_;
int column_;
std::vector< CompletionData > completion_datas_;
mutable boost::shared_mutex access_mutex_;
};
} // namespace YouCompleteMe
#endif /* end of include guard: CLANGRESULTSCACHE_H_REUWM3RU */

7
cpp/ycm/ClangCompleter/TranslationUnitStore.cpp
View File

@ -38,14 +38,17 @@ std::size_t HashForFlags( const std::vector< std::string > &flags ) {
} // unnamed namespace
TranslationUnitStore::TranslationUnitStore( CXIndex clang_index )
: clang_index_( clang_index ) {
}
TranslationUnitStore::~TranslationUnitStore() {
RemoveAll();
}
shared_ptr< TranslationUnit > TranslationUnitStore::GetOrCreate(
const std::string &filename,
const std::vector< UnsavedFile > &unsaved_files,
@ -104,24 +107,28 @@ shared_ptr< TranslationUnit > TranslationUnitStore::GetOrCreate(
return unit;
}
shared_ptr< TranslationUnit > TranslationUnitStore::Get(
const std::string &filename ) {
lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ );
return GetNoLock( filename );
}
bool TranslationUnitStore::Remove( const std::string &filename ) {
lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ );
Erase( filename_to_flags_hash_, filename );
return Erase( filename_to_translation_unit_, filename );
}
void TranslationUnitStore::RemoveAll() {
lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ );
filename_to_translation_unit_.clear();
filename_to_flags_hash_.clear();
}
shared_ptr< TranslationUnit > TranslationUnitStore::GetNoLock(
const std::string &filename ) {
return FindWithDefault( filename_to_translation_unit_,

152
cpp/ycm/IdentifierCompleter.cpp
View File

@ -23,62 +23,16 @@
#include "Result.h"
#include "Utils.h"
#include <boost/bind.hpp>
#include <boost/make_shared.hpp>
#include <algorithm>
using boost::packaged_task;
using boost::unique_future;
using boost::shared_ptr;
using boost::thread;
namespace YouCompleteMe {
typedef boost::function< std::vector< std::string >() >
FunctionReturnsStringVector;
extern const unsigned int MAX_ASYNC_THREADS = 4;
extern const unsigned int MIN_ASYNC_THREADS = 2;
namespace {
void QueryThreadMain(
IdentifierCompleter::LatestQueryTask &latest_query_task ) {
while ( true ) {
try {
( *latest_query_task.Get() )();
} catch ( boost::thread_interrupted & ) {
return;
}
}
}
void BufferIdentifiersThreadMain(
IdentifierCompleter::BufferIdentifiersTaskStack
&buffer_identifiers_task_stack ) {
while ( true ) {
try {
( *buffer_identifiers_task_stack.Pop() )();
} catch ( boost::thread_interrupted & ) {
return;
}
}
}
} // unnamed namespace
IdentifierCompleter::IdentifierCompleter()
: threading_enabled_( false ) {
}
IdentifierCompleter::IdentifierCompleter() {}
IdentifierCompleter::IdentifierCompleter(
const std::vector< std::string > &candidates )
: threading_enabled_( false ) {
const std::vector< std::string > &candidates ) {
identifier_database_.AddIdentifiers( candidates, "", "" );
}
@ -86,31 +40,11 @@ IdentifierCompleter::IdentifierCompleter(
IdentifierCompleter::IdentifierCompleter(
const std::vector< std::string > &candidates,
const std::string &filetype,
const std::string &filepath )
: threading_enabled_( false ) {
const std::string &filepath ) {
identifier_database_.AddIdentifiers( candidates, filetype, filepath );
}
IdentifierCompleter::~IdentifierCompleter() {
query_threads_.interrupt_all();
query_threads_.join_all();
if ( buffer_identifiers_thread_ ) {
buffer_identifiers_thread_->interrupt();
buffer_identifiers_thread_->join();
}
}
// We need this mostly so that we can not use it in tests. Apparently the
// GoogleTest framework goes apeshit on us if we enable threads by default.
void IdentifierCompleter::EnableThreading() {
threading_enabled_ = true;
InitThreads();
}
void IdentifierCompleter::AddIdentifiersToDatabase(
const std::vector< std::string > &new_candidates,
const std::string &filetype,
@ -130,22 +64,6 @@ void IdentifierCompleter::AddIdentifiersToDatabaseFromTagFiles(
}
void IdentifierCompleter::AddIdentifiersToDatabaseFromTagFilesAsync(
std::vector< std::string > absolute_paths_to_tag_files ) {
// TODO: throw exception when threading is not enabled and this is called
if ( !threading_enabled_ )
return;
boost::function< void() > functor =
boost::bind( &IdentifierCompleter::AddIdentifiersToDatabaseFromTagFiles,
boost::ref( *this ),
boost::move( absolute_paths_to_tag_files ) );
buffer_identifiers_task_stack_.Push(
boost::make_shared< packaged_task< void > >( boost::move( functor ) ) );
}
void IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer(
const std::string &buffer_contents,
const std::string &filetype,
@ -165,28 +83,6 @@ void IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer(
}
void IdentifierCompleter::AddIdentifiersToDatabaseFromBufferAsync(
std::string buffer_contents,
std::string filetype,
std::string filepath,
bool collect_from_comments_and_strings ) {
// TODO: throw exception when threading is not enabled and this is called
if ( !threading_enabled_ )
return;
boost::function< void() > functor =
boost::bind( &IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer,
boost::ref( *this ),
boost::move( buffer_contents ),
boost::move( filetype ),
boost::move( filepath ),
collect_from_comments_and_strings );
buffer_identifiers_task_stack_.Push(
boost::make_shared< packaged_task< void > >( boost::move( functor ) ) );
}
std::vector< std::string > IdentifierCompleter::CandidatesForQuery(
const std::string &query ) const {
return CandidatesForQueryAndType( query, "" );
@ -209,46 +105,4 @@ std::vector< std::string > IdentifierCompleter::CandidatesForQueryAndType(
}
Future< AsyncResults > IdentifierCompleter::CandidatesForQueryAndTypeAsync(
const std::string &query,
const std::string &filetype ) const {
// TODO: throw exception when threading is not enabled and this is called
if ( !threading_enabled_ )
return Future< AsyncResults >();
FunctionReturnsStringVector functor =
boost::bind( &IdentifierCompleter::CandidatesForQueryAndType,
boost::cref( *this ),
query,
filetype );
QueryTask task =
boost::make_shared< packaged_task< AsyncResults > >(
boost::bind( ReturnValueAsShared< std::vector< std::string > >,
boost::move( functor ) ) );
unique_future< AsyncResults > future = task->get_future();
latest_query_task_.Set( task );
return Future< AsyncResults >( boost::move( future ) );
}
void IdentifierCompleter::InitThreads() {
int query_threads_to_create =
std::max( MIN_ASYNC_THREADS,
std::min( MAX_ASYNC_THREADS, thread::hardware_concurrency() ) );
for ( int i = 0; i < query_threads_to_create; ++i ) {
query_threads_.create_thread(
boost::bind( QueryThreadMain,
boost::ref( latest_query_task_ ) ) );
}
buffer_identifiers_thread_.reset(
new boost::thread( BufferIdentifiersThreadMain,
boost::ref( buffer_identifiers_task_stack_ ) ) );
}
} // namespace YouCompleteMe

43
cpp/ycm/IdentifierCompleter.h
View File

@ -36,8 +36,6 @@ namespace YouCompleteMe {
class Candidate;
typedef boost::shared_ptr< std::vector< std::string > > AsyncResults;
class IdentifierCompleter : boost::noncopyable {
public:
@ -47,10 +45,6 @@ public:
const std::string &filetype,
const std::string &filepath );
~IdentifierCompleter();
void EnableThreading();
void AddIdentifiersToDatabase(
const std::vector< std::string > &new_candidates,
const std::string &filetype,
@ -59,25 +53,12 @@ public:
void AddIdentifiersToDatabaseFromTagFiles(
const std::vector< std::string > &absolute_paths_to_tag_files );
// NOTE: params are taken by value on purpose!
void AddIdentifiersToDatabaseFromTagFilesAsync(
std::vector< std::string > absolute_paths_to_tag_files );
void AddIdentifiersToDatabaseFromBuffer(
const std::string &buffer_contents,
const std::string &filetype,
const std::string &filepath,
bool collect_from_comments_and_strings );
// NOTE: params are taken by value on purpose! With a C++11 compiler we can
// avoid an expensive copy of buffer_contents if the param is taken by value
// (move ctors FTW)
void AddIdentifiersToDatabaseFromBufferAsync(
std::string buffer_contents,
std::string filetype,
std::string filepath,
bool collect_from_comments_and_strings );
// Only provided for tests!
std::vector< std::string > CandidatesForQuery(
const std::string &query ) const;
@ -86,37 +67,13 @@ public:
const std::string &query,
const std::string &filetype ) const;
Future< AsyncResults > CandidatesForQueryAndTypeAsync(
const std::string &query,
const std::string &filetype ) const;
typedef boost::shared_ptr <
boost::packaged_task< AsyncResults > > QueryTask;
typedef ConcurrentLatestValue< QueryTask > LatestQueryTask;
typedef ConcurrentStack< VoidTask > BufferIdentifiersTaskStack;
private:
void InitThreads();
/////////////////////////////
// PRIVATE MEMBER VARIABLES
/////////////////////////////
IdentifierDatabase identifier_database_;
bool threading_enabled_;
boost::thread_group query_threads_;
boost::scoped_ptr< boost::thread > buffer_identifiers_thread_;
mutable LatestQueryTask latest_query_task_;
BufferIdentifiersTaskStack buffer_identifiers_task_stack_;
};
} // namespace YouCompleteMe

18
cpp/ycm/tests/ClangCompleter/ClangCompleter_test.cpp
View File

@ -43,24 +43,6 @@ TEST( ClangCompleterTest, CandidatesForLocationInFile ) {
}
TEST( ClangCompleterTest, CandidatesForQueryAndLocationInFileAsync ) {
ClangCompleter completer;
completer.EnableThreading();
Future< AsyncCompletions > completions_future =
completer.CandidatesForQueryAndLocationInFileAsync(
"",
PathToTestFile( "basic.cpp" ).string(),
11,
7,
std::vector< UnsavedFile >(),
std::vector< std::string >() );
completions_future.Wait();
EXPECT_TRUE( !completions_future.GetResults()->empty() );
}
TEST( ClangCompleterTest, GetDefinitionLocation ) {
ClangCompleter completer;
std::string filename = PathToTestFile( "basic.cpp" ).string();

37
cpp/ycm/ycm_core.cpp
View File

@ -60,36 +60,24 @@ BOOST_PYTHON_MODULE(ycm_core)
def( "YcmCoreVersion", YcmCoreVersion );
class_< IdentifierCompleter, boost::noncopyable >( "IdentifierCompleter" )
.def( "EnableThreading", &IdentifierCompleter::EnableThreading )
.def( "AddIdentifiersToDatabase",
&IdentifierCompleter::AddIdentifiersToDatabase )
.def( "AddIdentifiersToDatabaseFromTagFilesAsync",
&IdentifierCompleter::AddIdentifiersToDatabaseFromTagFilesAsync )
.def( "AddIdentifiersToDatabaseFromBufferAsync",
&IdentifierCompleter::AddIdentifiersToDatabaseFromBufferAsync )
.def( "CandidatesForQueryAndTypeAsync",
&IdentifierCompleter::CandidatesForQueryAndTypeAsync );
.def( "AddIdentifiersToDatabaseFromTagFiles",
&IdentifierCompleter::AddIdentifiersToDatabaseFromTagFiles )
.def( "AddIdentifiersToDatabaseFromBuffer",
&IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer )
.def( "CandidatesForQueryAndType",
&IdentifierCompleter::CandidatesForQueryAndType );
// TODO: rename these *Vec classes to *Vector; don't forget the python file
class_< std::vector< std::string >,
boost::shared_ptr< std::vector< std::string > > >( "StringVec" )
.def( vector_indexing_suite< std::vector< std::string > >() );
class_< Future< AsyncResults > >( "FutureResults" )
.def( "ResultsReady", &Future< AsyncResults >::ResultsReady )
.def( "GetResults", &Future< AsyncResults >::GetResults );
class_< Future< void > >( "FutureVoid" )
.def( "ResultsReady", &Future< void >::ResultsReady )
.def( "GetResults", &Future< void >::GetResults );
#ifdef USE_CLANG_COMPLETER
def( "ClangVersion", ClangVersion );
class_< Future< AsyncCompletions > >( "FutureCompletions" )
.def( "ResultsReady", &Future< AsyncCompletions >::ResultsReady )
.def( "GetResults", &Future< AsyncCompletions >::GetResults );
// TODO: We may not need this at all anymore. Look into it.
class_< Future< AsyncCompilationInfoForFile > >(
"FutureCompilationInfoForFile" )
.def( "ResultsReady",
@ -116,17 +104,14 @@ BOOST_PYTHON_MODULE(ycm_core)
.def( vector_indexing_suite< std::vector< UnsavedFile > >() );
class_< ClangCompleter, boost::noncopyable >( "ClangCompleter" )
.def( "EnableThreading", &ClangCompleter::EnableThreading )
.def( "DiagnosticsForFile", &ClangCompleter::DiagnosticsForFile )
.def( "GetDeclarationLocation", &ClangCompleter::GetDeclarationLocation )
.def( "GetDefinitionLocation", &ClangCompleter::GetDefinitionLocation )
.def( "DeleteCachesForFileAsync",
&ClangCompleter::DeleteCachesForFileAsync )
.def( "DeleteCachesForFile", &ClangCompleter::DeleteCachesForFile )
.def( "UpdatingTranslationUnit", &ClangCompleter::UpdatingTranslationUnit )
.def( "UpdateTranslationUnitAsync",
&ClangCompleter::UpdateTranslationUnitAsync )
.def( "CandidatesForQueryAndLocationInFileAsync",
&ClangCompleter::CandidatesForQueryAndLocationInFileAsync );
.def( "UpdateTranslationUnit", &ClangCompleter::UpdateTranslationUnit )
.def( "CandidatesForLocationInFile",
&ClangCompleter::CandidatesForLocationInFile );
class_< CompletionData >( "CompletionData" )
.def( "TextToInsertInBuffer", &CompletionData::TextToInsertInBuffer )

2
python/test_requirements.txt
View File

@ -1,3 +1,5 @@
flake8>=2.0
mock>=1.0.1
nose>=1.3.0
PyHamcrest>=1.7.2

42
python/ycm/completers/all/identifier_completer.py
View File

@ -34,9 +34,8 @@ SYNTAX_FILENAME = 'YCM_PLACEHOLDER_FOR_SYNTAX'
class IdentifierCompleter( GeneralCompleter ):
def __init__( self, user_options ):
super( IdentifierCompleter, self ).__init__( user_options )
self.completer = ycm_core.IdentifierCompleter()
self.completer.EnableThreading()
self.tags_file_last_mtime = defaultdict( int )
self._completer = ycm_core.IdentifierCompleter()
self._tags_file_last_mtime = defaultdict( int )
self._logger = logging.getLogger( __name__ )
@ -44,11 +43,20 @@ class IdentifierCompleter( GeneralCompleter ):
return self.QueryLengthAboveMinThreshold( request_data )
def CandidatesForQueryAsync( self, request_data ):
self.completions_future = self.completer.CandidatesForQueryAndTypeAsync(
def ComputeCandidates( self, request_data ):
if not self.ShouldUseNow( request_data ):
return []
completions = self._completer.CandidatesForQueryAndType(
ToUtf8IfNeeded( utils.SanitizeQuery( request_data[ 'query' ] ) ),
ToUtf8IfNeeded( request_data[ 'filetypes' ][ 0 ] ) )
completions = completions[ : MAX_IDENTIFIER_COMPLETIONS_RETURNED ]
completions = _RemoveSmallCandidates(
completions, self.user_options[ 'min_num_identifier_candidate_chars' ] )
return [ responses.BuildCompletionData( x ) for x in completions ]
def AddIdentifier( self, identifier, request_data ):
filetype = request_data[ 'filetypes' ][ 0 ]
@ -60,7 +68,7 @@ class IdentifierCompleter( GeneralCompleter ):
vector = ycm_core.StringVec()
vector.append( ToUtf8IfNeeded( identifier ) )
self._logger.info( 'Adding ONE buffer identifier for file: %s', filepath )
self.completer.AddIdentifiersToDatabase( vector,
self._completer.AddIdentifiersToDatabase( vector,
ToUtf8IfNeeded( filetype ),
ToUtf8IfNeeded( filepath ) )
@ -92,7 +100,7 @@ class IdentifierCompleter( GeneralCompleter ):
text = request_data[ 'file_data' ][ filepath ][ 'contents' ]
self._logger.info( 'Adding buffer identifiers for file: %s', filepath )
self.completer.AddIdentifiersToDatabaseFromBufferAsync(
self._completer.AddIdentifiersToDatabaseFromBuffer(
ToUtf8IfNeeded( text ),
ToUtf8IfNeeded( filetype ),
ToUtf8IfNeeded( filepath ),
@ -106,20 +114,20 @@ class IdentifierCompleter( GeneralCompleter ):
current_mtime = os.path.getmtime( tag_file )
except:
continue
last_mtime = self.tags_file_last_mtime[ tag_file ]
last_mtime = self._tags_file_last_mtime[ tag_file ]
# We don't want to repeatedly process the same file over and over; we only
# process if it's changed since the last time we looked at it
if current_mtime <= last_mtime:
continue
self.tags_file_last_mtime[ tag_file ] = current_mtime
self._tags_file_last_mtime[ tag_file ] = current_mtime
absolute_paths_to_tag_files.append( ToUtf8IfNeeded( tag_file ) )
if not absolute_paths_to_tag_files:
return
self.completer.AddIdentifiersToDatabaseFromTagFilesAsync(
self._completer.AddIdentifiersToDatabaseFromTagFiles(
absolute_paths_to_tag_files )
@ -129,7 +137,7 @@ class IdentifierCompleter( GeneralCompleter ):
keyword_vector.append( ToUtf8IfNeeded( keyword ) )
filepath = SYNTAX_FILENAME + filetypes[ 0 ]
self.completer.AddIdentifiersToDatabase( keyword_vector,
self._completer.AddIdentifiersToDatabase( keyword_vector,
ToUtf8IfNeeded( filetypes[ 0 ] ),
ToUtf8IfNeeded( filepath ) )
@ -151,18 +159,6 @@ class IdentifierCompleter( GeneralCompleter ):
self.AddPreviousIdentifier( request_data )
def CandidatesFromStoredRequest( self ):
if not self.completions_future:
return []
completions = self.completions_future.GetResults()[
: MAX_IDENTIFIER_COMPLETIONS_RETURNED ]
completions = _RemoveSmallCandidates(
completions, self.user_options[ 'min_num_identifier_candidate_chars' ] )
return [ responses.BuildCompletionData( x ) for x in completions ]
def _PreviousIdentifier( min_num_completion_start_chars, request_data ):
line_num = request_data[ 'line_num' ]
column_num = request_data[ 'column_num' ]

117
python/ycm/completers/completer.py
View File

@ -58,44 +58,21 @@ class Completer( object ):
and will NOT re-query your completer but will in fact provide fuzzy-search on
the candidate strings that were stored in the cache.
CandidatesForQueryAsync() is the main entry point when the user types. For
ComputeCandidates() is the main entry point when the user types. For
"foo.bar", the user query is "bar" and completions matching this string should
be shown. The job of CandidatesForQueryAsync() is to merely initiate this
request, which will hopefully be processed in a background thread. You may
want to subclass ThreadedCompleter instead of Completer directly.
be shown. It should return the list of candidates. The format of the result
can be a list of strings or a more complicated list of dictionaries. Use
ycm.server.responses.BuildCompletionData to build the detailed response. See
clang_completer.py to see how its used in practice.
AsyncCandidateRequestReady() is the function that is repeatedly polled until
it returns True. If CandidatesForQueryAsync() started a background task of
collecting the required completions, AsyncCandidateRequestReady() would check
the state of that task and return False until it was completed.
CandidatesFromStoredRequest() should return the list of candidates. This is
what YCM calls after AsyncCandidateRequestReady() returns True. The format of
the result can be a list of strings or a more complicated list of
dictionaries. See ':h complete-items' for the format, and clang_completer.py
to see how its used in practice.
Again, you probably want to override ComputeCandidatesInner().
You also need to implement the SupportedFiletypes() function which should
return a list of strings, where the strings are Vim filetypes your completer
supports.
clang_completer.py is a good example of a "complicated" completer that
maintains its own internal cache and therefore directly overrides the "main"
functions in the API instead of the *Inner versions. A good example of a
simple completer that does not do this is omni_completer.py.
If you're confident your completer doesn't need a background task (think
again, you probably do) because you can "certainly" furnish a response in
under 10ms, then you can perform your backend processing in a synchronous
fashion. You may also need to do this because of technical restrictions (much
like omni_completer.py has to do it because accessing Vim internals is not
thread-safe). But even if you're certain, still try to do the processing in a
background thread. Your completer is unlikely to be merged if it does not,
because synchronous processing will block Vim's GUI thread and that's a very,
VERY bad thing (so try not to do it!). Again, you may want to subclass
ThreadedCompleter instead of Completer directly; ThreadedCompleter will
abstract away the use of a background thread for you. See
threaded_completer.py.
clang_completer.py is a good example of a "complicated" completer that A good
example of a simple completer is ultisnips_completer.py.
The On* functions are provided for your convenience. They are called when
their specific events occur. For instance, the identifier completer collects
@ -122,7 +99,7 @@ class Completer( object ):
self.triggers_for_filetype = TriggersForFiletype(
user_options[ 'semantic_triggers' ] )
self.completions_future = None
self.completions_cache = None
self._completions_cache = None
# It's highly likely you DON'T want to override this function but the *Inner
@ -130,13 +107,13 @@ class Completer( object ):
def ShouldUseNow( self, request_data ):
inner_says_yes = self.ShouldUseNowInner( request_data )
if not inner_says_yes:
self.completions_cache = None
self._completions_cache = None
previous_results_were_empty = ( self.completions_cache and
self.completions_cache.CacheValid(
previous_results_were_empty = ( self._completions_cache and
self._completions_cache.CacheValid(
request_data[ 'line_num' ],
request_data[ 'start_column' ] ) and
not self.completions_cache.raw_completions )
not self._completions_cache.raw_completions )
return inner_says_yes and not previous_results_were_empty
@ -171,20 +148,28 @@ class Completer( object ):
# It's highly likely you DON'T want to override this function but the *Inner
# version of it.
def CandidatesForQueryAsync( self, request_data ):
self.request_data = request_data
def ComputeCandidates( self, request_data ):
if not self.ShouldUseNow( request_data ):
return []
if ( request_data[ 'query' ] and
self.completions_cache and
self.completions_cache.CacheValid( request_data[ 'line_num' ],
self._completions_cache and
self._completions_cache.CacheValid( request_data[ 'line_num' ],
request_data[ 'start_column' ] ) ):
self.completions_cache.filtered_completions = (
self.FilterAndSortCandidates(
self.completions_cache.raw_completions,
request_data[ 'query' ] ) )
return self.FilterAndSortCandidates(
self._completions_cache.raw_completions,
request_data[ 'query' ] )
else:
self.completions_cache = None
self.CandidatesForQueryAsyncInner( request_data )
self._completions_cache = CompletionsCache()
self._completions_cache.raw_completions = self.ComputeCandidatesInner(
request_data )
self._completions_cache.line = request_data[ 'line_num' ]
self._completions_cache.column = request_data[ 'start_column' ]
return self._completions_cache.raw_completions
def ComputeCandidatesInner( self, request_data ):
pass
def DefinedSubcommands( self ):
@ -224,46 +209,6 @@ class Completer( object ):
return matches
def CandidatesForQueryAsyncInner( self, query, start_column ):
pass
# It's highly likely you DON'T want to override this function but the *Inner
# version of it.
def AsyncCandidateRequestReady( self ):
if self.completions_cache:
return True
else:
return self.AsyncCandidateRequestReadyInner()
def AsyncCandidateRequestReadyInner( self ):
if not self.completions_future:
# We return True so that the caller can extract the default value from the
# future
return True
return self.completions_future.ResultsReady()
# It's highly likely you DON'T want to override this function but the *Inner
# version of it.
def CandidatesFromStoredRequest( self ):
if self.completions_cache:
return self.completions_cache.filtered_completions
else:
self.completions_cache = CompletionsCache()
self.completions_cache.raw_completions = self.CandidatesFromStoredRequestInner()
self.completions_cache.line = self.request_data[ 'line_num' ]
self.completions_cache.column = self.request_data[ 'start_column' ]
return self.completions_cache.raw_completions
def CandidatesFromStoredRequestInner( self ):
if not self.completions_future:
return []
return self.completions_future.GetResults()
def OnFileReadyToParse( self, request_data ):
pass

116
python/ycm/completers/cpp/clang_completer.py
View File

@ -30,8 +30,8 @@ CLANG_FILETYPES = set( [ 'c', 'cpp', 'objc', 'objcpp' ] )
MIN_LINES_IN_FILE_TO_PARSE = 5
PARSING_FILE_MESSAGE = 'Still parsing file, no completions yet.'
NO_COMPILE_FLAGS_MESSAGE = 'Still no compile flags, no completions yet.'
NO_COMPLETIONS_MESSAGE = 'No completions found; errors in the file?'
INVALID_FILE_MESSAGE = 'File is invalid.'
NO_COMPLETIONS_MESSAGE = 'No completions found; errors in the file?'
FILE_TOO_SHORT_MESSAGE = (
'File is less than {} lines long; not compiling.'.format(
MIN_LINES_IN_FILE_TO_PARSE ) )
@ -41,25 +41,14 @@ NO_DIAGNOSTIC_MESSAGE = 'No diagnostic for current line!'
class ClangCompleter( Completer ):
def __init__( self, user_options ):
super( ClangCompleter, self ).__init__( user_options )
self.max_diagnostics_to_display = user_options[
self._max_diagnostics_to_display = user_options[
'max_diagnostics_to_display' ]
self.completer = ycm_core.ClangCompleter()
self.completer.EnableThreading()
self.last_prepared_diagnostics = []
self.parse_future = None
self.flags = Flags()
self.diagnostic_store = None
self._completer = ycm_core.ClangCompleter()
self._last_prepared_diagnostics = []
self._flags = Flags()
self._diagnostic_store = None
self._logger = logging.getLogger( __name__ )
# We set this flag when a compilation request comes in while one is already
# in progress. We use this to trigger the pending request after the previous
# one completes (from GetDiagnosticsForCurrentFile because that's the only
# method that knows when the compilation has finished).
# TODO: Remove this now that we have multiple threads in the server; the
# subsequent requests that want to parse will just block until the current
# parse is done and will then proceed.
self.extra_parse_desired = False
def SupportedFiletypes( self ):
return CLANG_FILETYPES
@ -84,53 +73,35 @@ class ClangCompleter( Completer ):
return files
def CandidatesForQueryAsync( self, request_data ):
def ComputeCandidatesInner( self, request_data ):
filename = request_data[ 'filepath' ]
if not filename:
return
if self.completer.UpdatingTranslationUnit( ToUtf8IfNeeded( filename ) ):
self.completions_future = None
if self._completer.UpdatingTranslationUnit( ToUtf8IfNeeded( filename ) ):
self._logger.info( PARSING_FILE_MESSAGE )
return responses.BuildDisplayMessageResponse(
PARSING_FILE_MESSAGE )
raise RuntimeError( PARSING_FILE_MESSAGE )
flags = self._FlagsForRequest( request_data )
if not flags:
self.completions_future = None
self._logger.info( NO_COMPILE_FLAGS_MESSAGE )
return responses.BuildDisplayMessageResponse(
NO_COMPILE_FLAGS_MESSAGE )
raise RuntimeError( NO_COMPILE_FLAGS_MESSAGE )
# TODO: sanitize query, probably in C++ code
files = ycm_core.UnsavedFileVec()
query = request_data[ 'query' ]
if not query:
files = self.GetUnsavedFilesVector( request_data )
line = request_data[ 'line_num' ] + 1
column = request_data[ 'start_column' ] + 1
self.completions_future = (
self.completer.CandidatesForQueryAndLocationInFileAsync(
ToUtf8IfNeeded( query ),
results = self._completer.CandidatesForLocationInFile(
ToUtf8IfNeeded( filename ),
line,
column,
files,
flags ) )
flags )
def CandidatesFromStoredRequest( self ):
if not self.completions_future:
return []
results = [ ConvertCompletionData( x ) for x in
self.completions_future.GetResults() ]
if not results:
self._logger.warning( NO_COMPLETIONS_MESSAGE )
raise RuntimeError( NO_COMPLETIONS_MESSAGE )
return results
return [ ConvertCompletionData( x ) for x in results ]
def DefinedSubcommands( self ):
@ -170,7 +141,7 @@ class ClangCompleter( Completer ):
files = self.GetUnsavedFilesVector( request_data )
line = request_data[ 'line_num' ] + 1
column = request_data[ 'column_num' ] + 1
return getattr( self.completer, goto_function )(
return getattr( self._completer, goto_function )(
ToUtf8IfNeeded( filename ),
line,
column,
@ -212,73 +183,63 @@ class ClangCompleter( Completer ):
def _ClearCompilationFlagCache( self ):
self.flags.Clear()
self._flags.Clear()
def OnFileReadyToParse( self, request_data ):
filename = request_data[ 'filepath' ]
contents = request_data[ 'file_data' ][ filename ][ 'contents' ]
if contents.count( '\n' ) < MIN_LINES_IN_FILE_TO_PARSE:
self.parse_future = None
self._logger.warning( FILE_TOO_SHORT_MESSAGE )
raise ValueError( FILE_TOO_SHORT_MESSAGE )
if not filename:
self._logger.warning( INVALID_FILE_MESSAGE )
return responses.BuildDisplayMessageResponse(
INVALID_FILE_MESSAGE )
if self.completer.UpdatingTranslationUnit( ToUtf8IfNeeded( filename ) ):
self.extra_parse_desired = True
return
raise ValueError( INVALID_FILE_MESSAGE )
flags = self._FlagsForRequest( request_data )
if not flags:
self.parse_future = None
self._logger.info( NO_COMPILE_FLAGS_MESSAGE )
return responses.BuildDisplayMessageResponse(
NO_COMPILE_FLAGS_MESSAGE )
raise ValueError( NO_COMPILE_FLAGS_MESSAGE )
self.parse_future = self.completer.UpdateTranslationUnitAsync(
self._completer.UpdateTranslationUnit(
ToUtf8IfNeeded( filename ),
self.GetUnsavedFilesVector( request_data ),
flags )
self.extra_parse_desired = False
def OnBufferUnload( self, request_data ):
self.completer.DeleteCachesForFileAsync(
self._completer.DeleteCachesForFile(
ToUtf8IfNeeded( request_data[ 'unloaded_buffer' ] ) )
def DiagnosticsForCurrentFileReady( self ):
if not self.parse_future:
return False
return self.parse_future.ResultsReady()
# if not self.parse_future:
# return False
# return self.parse_future.ResultsReady()
pass
def GettingCompletions( self, request_data ):
return self.completer.UpdatingTranslationUnit(
return self._completer.UpdatingTranslationUnit(
ToUtf8IfNeeded( request_data[ 'filepath' ] ) )
def GetDiagnosticsForCurrentFile( self, request_data ):
filename = request_data[ 'filepath' ]
if self.DiagnosticsForCurrentFileReady():
diagnostics = self.completer.DiagnosticsForFile(
diagnostics = self._completer.DiagnosticsForFile(
ToUtf8IfNeeded( filename ) )
self.diagnostic_store = DiagnosticsToDiagStructure( diagnostics )
self.last_prepared_diagnostics = [
self._diagnostic_store = DiagnosticsToDiagStructure( diagnostics )
self._last_prepared_diagnostics = [
responses.BuildDiagnosticData( x ) for x in
diagnostics[ : self.max_diagnostics_to_display ] ]
self.parse_future = None
diagnostics[ : self._max_diagnostics_to_display ] ]
# self.parse_future = None
if self.extra_parse_desired:
self.OnFileReadyToParse( request_data )
# if self.extra_parse_desired:
# self.OnFileReadyToParse( request_data )
return self.last_prepared_diagnostics
return self._last_prepared_diagnostics
def GetDetailedDiagnostic( self, request_data ):
@ -286,11 +247,11 @@ class ClangCompleter( Completer ):
current_column = request_data[ 'column_num' ] + 1
current_file = request_data[ 'filepath' ]
if not self.diagnostic_store:
if not self._diagnostic_store:
return responses.BuildDisplayMessageResponse(
NO_DIAGNOSTIC_MESSAGE )
diagnostics = self.diagnostic_store[ current_file ][ current_line ]
diagnostics = self._diagnostic_store[ current_file ][ current_line ]
if not diagnostics:
return responses.BuildDisplayMessageResponse(
NO_DIAGNOSTIC_MESSAGE )
@ -308,11 +269,6 @@ class ClangCompleter( Completer ):
closest_diagnostic.long_formatted_text_ )
def ShouldUseNow( self, request_data ):
# We don't want to use the Completer API cache, we use one in the C++ code.
return self.ShouldUseNowInner( request_data )
def DebugInfo( self, request_data ):
filename = request_data[ 'filepath' ]
if not filename:
@ -328,7 +284,7 @@ class ClangCompleter( Completer ):
if 'compilation_flags' in request_data:
return PrepareFlagsForClang( request_data[ 'compilation_flags' ],
filename )
return self.flags.FlagsForFile( filename )
return self._flags.FlagsForFile( filename )
# TODO: Make this work again
# def DiagnosticToDict( diagnostic ):

6
python/ycm/completers/cs/cs_completer.py
View File

@ -21,7 +21,7 @@
import os
from sys import platform
import glob
from ycm.completers.threaded_completer import ThreadedCompleter
from ycm.completers.completer import Completer
from ycm.server import responses
from ycm import utils
import urllib2
@ -36,7 +36,7 @@ SERVER_NOT_FOUND_MSG = ( 'OmniSharp server binary not found at {0}. ' +
'Did you compile it? You can do so by running ' +
'"./install.sh --omnisharp-completer".' )
class CsharpCompleter( ThreadedCompleter ):
class CsharpCompleter( Completer ):
"""
A Completer that uses the Omnisharp server as completion engine.
"""
@ -61,7 +61,7 @@ class CsharpCompleter( ThreadedCompleter ):
return [ 'cs' ]
def ComputeCandidates( self, request_data ):
def ComputeCandidatesInner( self, request_data ):
return [ responses.BuildCompletionData(
completion[ 'CompletionText' ],
completion[ 'DisplayText' ],

6
python/ycm/completers/general/filename_completer.py
View File

@ -19,12 +19,12 @@
import os
import re
from ycm.completers.threaded_completer import ThreadedCompleter
from ycm.completers.completer import Completer
from ycm.completers.cpp.clang_completer import InCFamilyFile
from ycm.completers.cpp.flags import Flags
from ycm.server import responses
class FilenameCompleter( ThreadedCompleter ):
class FilenameCompleter( Completer ):
"""
General completer that provides filename and filepath completions.
"""
@ -73,7 +73,7 @@ class FilenameCompleter( ThreadedCompleter ):
return []
def ComputeCandidates( self, request_data ):
def ComputeCandidatesInner( self, request_data ):
current_line = request_data[ 'line_value' ]
start_column = request_data[ 'start_column' ]
filepath = request_data[ 'filepath' ]

15
python/ycm/completers/general/general_completer_store.py
View File

@ -74,20 +74,13 @@ class GeneralCompleterStore( Completer ):
return should_use_now
def CandidatesForQueryAsync( self, request_data ):
for completer in self._current_query_completers:
completer.CandidatesForQueryAsync( request_data )
def ComputeCandidates( self, request_data ):
if not self.ShouldUseNow( request_data ):
return []
def AsyncCandidateRequestReady( self ):
return all( x.AsyncCandidateRequestReady() for x in
self._current_query_completers )
def CandidatesFromStoredRequest( self ):
candidates = []
for completer in self._current_query_completers:
candidates += completer.CandidatesFromStoredRequest()
candidates += completer.ComputeCandidates( request_data )
return candidates

16
python/ycm/completers/general/ultisnips_completer.py
View File

@ -33,23 +33,17 @@ class UltiSnipsCompleter( GeneralCompleter ):
self._filtered_candidates = None
def ShouldUseNowInner( self, request_data ):
def ShouldUseNow( self, request_data ):
return self.QueryLengthAboveMinThreshold( request_data )
def CandidatesForQueryAsync( self, request_data ):
self._filtered_candidates = self.FilterAndSortCandidates(
def ComputeCandidates( self, request_data ):
if not self.ShouldUseNow( request_data ):
return []
return self.FilterAndSortCandidates(
self._candidates, request_data[ 'query' ] )
def AsyncCandidateRequestReady( self ):
return True
def CandidatesFromStoredRequest( self ):
return self._filtered_candidates if self._filtered_candidates else []
def OnBufferVisit( self, request_data ):
raw_candidates = request_data[ 'ultisnips_snippets' ]
self._candidates = [

6
python/ycm/completers/python/jedi_completer.py
View File

@ -19,7 +19,7 @@
# You should have received a copy of the GNU General Public License
# along with YouCompleteMe. If not, see <http://www.gnu.org/licenses/>.
from ycm.completers.threaded_completer import ThreadedCompleter
from ycm.completers.completer import Completer
from ycm.server import responses
import sys
@ -38,7 +38,7 @@ except ImportError:
sys.path.pop( 0 )
class JediCompleter( ThreadedCompleter ):
class JediCompleter( Completer ):
"""
A Completer that uses the Jedi completion engine.
https://jedi.readthedocs.org/en/latest/
@ -63,7 +63,7 @@ class JediCompleter( ThreadedCompleter ):
return jedi.Script( contents, line, column, filename )
def ComputeCandidates( self, request_data ):
def ComputeCandidatesInner( self, request_data ):
script = self._GetJediScript( request_data )
return [ responses.BuildCompletionData(
str( completion.name ),

100
python/ycm/completers/threaded_completer.py
View File

@ -1,100 +0,0 @@
#!/usr/bin/env python
#
# Copyright (C) 2011, 2012 Strahinja Val Markovic <val@markovic.io>
#
# This file is part of YouCompleteMe.
#
# YouCompleteMe is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# YouCompleteMe is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with YouCompleteMe. If not, see <http://www.gnu.org/licenses/>.
import abc
from threading import Thread, Event
from ycm.completers.completer import Completer
class ThreadedCompleter( Completer ):
"""A subclass of Completer that abstracts away the use of a background thread.
This is a great class to subclass for your custom completer. It will provide
you with async computation of candidates when you implement the
ComputeCandidates() method; no need to worry about threads, locks, events or
similar.
If you use this class as the base class for your Completer, you DON'T need to
provide implementations for the CandidatesForQueryAsync(),
AsyncCandidateRequestReady() and CandidatesFromStoredRequest() functions. Just
implement ComputeCandidates().
For examples of subclasses of this class, see the following files:
python/completers/general/filename_completer.py
"""
def __init__( self, user_options ):
super( ThreadedCompleter, self ).__init__( user_options )
self._query_ready = Event()
self._candidates_ready = Event()
self._candidates = None
self._start_completion_thread()
def _start_completion_thread( self ):
self._completion_thread = Thread( target=self.SetCandidates )
self._completion_thread.daemon = True
self._completion_thread.start()
def CandidatesForQueryAsyncInner( self, request_data ):
self._candidates = None
self._candidates_ready.clear()
self._request_data = request_data
self._query_ready.set()
def AsyncCandidateRequestReadyInner( self ):
return WaitAndClearIfSet( self._candidates_ready, timeout=0.005 )
def CandidatesFromStoredRequestInner( self ):
return self._candidates or []
@abc.abstractmethod
def ComputeCandidates( self, request_data ):
"""This function should compute the candidates to show to the user.
The return value should be of the same type as that for
CandidatesFromStoredRequest()."""
pass
def SetCandidates( self ):
while True:
try:
WaitAndClearIfSet( self._query_ready )
self._candidates = self.ComputeCandidates( self._request_data )
except:
self._query_ready.clear()
self._candidates = []
self._candidates_ready.set()
def WaitAndClearIfSet( event, timeout=None ):
"""Given an |event| and a |timeout|, waits for the event a maximum of timeout
seconds. After waiting, clears the event if it's set and returns the state of
the event before it was cleared."""
# We can't just do flag_is_set = event.wait( timeout ) because that breaks on
# Python 2.6
event.wait( timeout )
flag_is_set = event.is_set()
if flag_is_set:
event.clear()
return flag_is_set

48
python/ycm/server/tests/basic_test.py
View File

@ -21,6 +21,7 @@ from webtest import TestApp
from .. import ycmd
from ..responses import BuildCompletionData
from nose.tools import ok_, eq_, with_setup
from hamcrest import assert_that, has_items, has_entry
import bottle
bottle.debug( True )
@ -41,6 +42,11 @@ def RequestDataForFileWithContents( filename, contents = None ):
}
# TODO: Make the other tests use this helper too instead of BuildCompletionData
def CompletionEntryMatcher( insertion_text ):
return has_entry( 'insertion_text', insertion_text )
def Setup():
ycmd.SetServerStateToDefaults()
@ -69,6 +75,48 @@ def GetCompletions_IdentifierCompleter_Works_test():
app.post_json( '/get_completions', completion_data ).json )
@with_setup( Setup )
def GetCompletions_ClangCompleter_Works_test():
app = TestApp( ycmd.app )
contents = """
struct Foo {
int x;
int y;
char c;
};
int main()
{
Foo foo;
foo.
}
"""
filename = '/foo.cpp'
completion_data = {
'compilation_flags': ['-x', 'c++'],
# 0-based line and column!
'query': '',
'line_num': 10,
'column_num': 6,
'start_column': 6,
'line_value': ' foo.',
'filetypes': ['cpp'],
'filepath': filename,
'file_data': {
filename: {
'contents': contents,
'filetypes': ['cpp']
}
}
}
results = app.post_json( '/get_completions', completion_data ).json
assert_that( results, has_items( CompletionEntryMatcher( 'c' ),
CompletionEntryMatcher( 'x' ),
CompletionEntryMatcher( 'y' ) ) )
@with_setup( Setup )
def GetCompletions_IdentifierCompleter_SyntaxKeywordsAdded_test():
app = TestApp( ycmd.app )

12
python/ycm/server/ycmd.py
View File

@ -31,7 +31,6 @@ sys.path.insert( 0, os.path.join(
'../..' ) )
import logging
import time
import json
import bottle
from bottle import run, request, response
@ -89,16 +88,7 @@ def GetCompletions():
do_filetype_completion else
SERVER_STATE.GetGeneralCompleter() )
# This is necessary so that general_completer_store fills up
# _current_query_completers.
# TODO: Fix this.
completer.ShouldUseNow( request_data )
# TODO: This should not be async anymore, server is multi-threaded
completer.CandidatesForQueryAsync( request_data )
while not completer.AsyncCandidateRequestReady():
time.sleep( 0.03 )
return _JsonResponse( completer.CandidatesFromStoredRequest() )
return _JsonResponse( completer.ComputeCandidates( request_data ) )
@app.get( '/user_options' )