From 6c53bad58f2c9d67167ca49ff536f381f2088653 Mon Sep 17 00:00:00 2001 From: Strahinja Val Markovic Date: Mon, 30 Sep 2013 13:40:25 -0700 Subject: [PATCH] No more threading in completers! The server is multi-threaded and will spawn a new thread for each new request. Thus, the completers need not manage their own threads or even provide async APIs; we _want_ them to block because now were implementing the request-response networking API. The client gets the async API through the network (i.e., it can do something else while the request is pending). --- cpp/ycm/ClangCompleter/ClangCompleter.cpp | 345 +----------------- cpp/ycm/ClangCompleter/ClangCompleter.h | 106 +----- cpp/ycm/ClangCompleter/ClangResultsCache.cpp | 43 --- cpp/ycm/ClangCompleter/ClangResultsCache.h | 84 ----- .../ClangCompleter/TranslationUnitStore.cpp | 7 + cpp/ycm/IdentifierCompleter.cpp | 152 +------- cpp/ycm/IdentifierCompleter.h | 43 --- .../ClangCompleter/ClangCompleter_test.cpp | 18 - cpp/ycm/ycm_core.cpp | 37 +- python/test_requirements.txt | 2 + .../completers/all/identifier_completer.py | 42 +-- python/ycm/completers/completer.py | 119 ++---- python/ycm/completers/cpp/clang_completer.py | 118 ++---- python/ycm/completers/cs/cs_completer.py | 6 +- .../completers/general/filename_completer.py | 6 +- .../general/general_completer_store.py | 15 +- .../completers/general/ultisnips_completer.py | 16 +- .../ycm/completers/python/jedi_completer.py | 6 +- python/ycm/completers/threaded_completer.py | 100 ----- python/ycm/server/tests/basic_test.py | 48 +++ python/ycm/server/ycmd.py | 12 +- 21 files changed, 189 insertions(+), 1136 deletions(-) delete mode 100644 cpp/ycm/ClangCompleter/ClangResultsCache.cpp delete mode 100644 cpp/ycm/ClangCompleter/ClangResultsCache.h delete mode 100644 python/ycm/completers/threaded_completer.py diff --git a/cpp/ycm/ClangCompleter/ClangCompleter.cpp b/cpp/ycm/ClangCompleter/ClangCompleter.cpp index 961de1d7..141af8ac 100644 --- a/cpp/ycm/ClangCompleter/ClangCompleter.cpp +++ b/cpp/ycm/ClangCompleter/ClangCompleter.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2011, 2012 Strahinja Val Markovic +// Copyright (C) 2011-2013 Strahinja Val Markovic // // This file is part of YouCompleteMe. // @@ -28,43 +28,17 @@ #include "ClangUtils.h" #include -#include -#include -#include +#include -using boost::algorithm::any_of; -using boost::algorithm::is_upper; -using boost::bind; -using boost::cref; -using boost::function; -using boost::lock_guard; -using boost::make_shared; -using boost::mutex; -using boost::packaged_task; -using boost::ref; -using boost::shared_lock; -using boost::shared_mutex; using boost::shared_ptr; -using boost::thread; -using boost::thread_interrupted; -using boost::try_to_lock_t; -using boost::unique_future; -using boost::unique_lock; using boost::unordered_map; namespace YouCompleteMe { -extern const unsigned int MAX_ASYNC_THREADS; -extern const unsigned int MIN_ASYNC_THREADS; - ClangCompleter::ClangCompleter() : clang_index_( clang_createIndex( 0, 0 ) ), - translation_unit_store_( clang_index_ ), - candidate_repository_( CandidateRepository::Instance() ), - threading_enabled_( false ), - time_to_die_( false ), - clang_data_ready_( false ) { + translation_unit_store_( clang_index_ ) { // The libclang docs don't say what is the default value for crash recovery. // I'm pretty sure it's turned on by default, but I'm not going to take any // chances. @@ -73,35 +47,18 @@ ClangCompleter::ClangCompleter() ClangCompleter::~ClangCompleter() { - { - unique_lock< shared_mutex > lock( time_to_die_mutex_ ); - time_to_die_ = true; - } - - sorting_threads_.interrupt_all(); - sorting_threads_.join_all(); - - if ( clang_thread_ ) { - clang_thread_->interrupt(); - clang_thread_->join(); - } - // We need to destroy all TUs before calling clang_disposeIndex because // the translation units need to be destroyed before the index is destroyed. + // Technically, a thread could still be holding onto a shared_ptr object + // when we destroy the clang index, but since we're shutting down, we don't + // really care. + // In practice, this situation shouldn't happen because the server threads are + // Python deamon threads and will all be killed before the main thread exits. translation_unit_store_.RemoveAll(); clang_disposeIndex( clang_index_ ); } -// We need this mostly so that we can not use it in tests. Apparently the -// GoogleTest framework goes apeshit on us (on some platforms, in some -// occasions) if we enable threads by default. -void ClangCompleter::EnableThreading() { - threading_enabled_ = true; - InitThreads(); -} - - std::vector< Diagnostic > ClangCompleter::DiagnosticsForFile( std::string filename ) { shared_ptr< TranslationUnit > unit = translation_unit_store_.Get( filename ); @@ -156,30 +113,6 @@ void ClangCompleter::UpdateTranslationUnit( } -Future< void > ClangCompleter::UpdateTranslationUnitAsync( - std::string filename, - std::vector< UnsavedFile > unsaved_files, - std::vector< std::string > flags ) { - function< void() > functor = - boost::bind( &ClangCompleter::UpdateTranslationUnit, - boost::ref( *this ), - boost::move( filename ), - boost::move( unsaved_files ), - boost::move( flags ) ); - - shared_ptr< ClangPackagedTask > clang_packaged_task = - make_shared< ClangPackagedTask >(); - - clang_packaged_task->parsing_task_ = packaged_task< void >( - boost::move( functor ) ); - unique_future< void > future = - clang_packaged_task->parsing_task_.get_future(); - clang_task_.Set( clang_packaged_task ); - - return Future< void >( boost::move( future ) ); -} - - std::vector< CompletionData > ClangCompleter::CandidatesForLocationInFile( std::string filename, @@ -199,56 +132,6 @@ ClangCompleter::CandidatesForLocationInFile( } -Future< AsyncCompletions > -ClangCompleter::CandidatesForQueryAndLocationInFileAsync( - std::string query, - std::string filename, - int line, - int column, - std::vector< UnsavedFile > unsaved_files, - std::vector< std::string > flags ) { - // TODO: throw exception when threading is not enabled and this is called - if ( !threading_enabled_ ) - return Future< AsyncCompletions >(); - - bool skip_clang_result_cache = ShouldSkipClangResultCache( query, - line, - column ); - - if ( skip_clang_result_cache ) { - // The clang thread is busy, return nothing - if ( UpdatingTranslationUnit( filename ) ) - return Future< AsyncCompletions >(); - - { - lock_guard< mutex > lock( clang_data_ready_mutex_ ); - clang_data_ready_ = false; - } - - // Needed to "reset" the sorting threads to the start of their loop. This - // way any threads blocking on a read in sorting_task_.Get() are reset to - // wait on the clang_data_ready_condition_variable_. - sorting_threads_.interrupt_all(); - } - - // the sorting task needs to be set before the clang task (if any) just in - // case the clang task finishes (and therefore notifies a sorting thread to - // consume a sorting task) before the sorting task is set - unique_future< AsyncCompletions > future; - CreateSortingTask( query, future ); - - if ( skip_clang_result_cache ) { - CreateClangTask( boost::move( filename ), - line, - column, - boost::move( unsaved_files ), - boost::move( flags ) ); - } - - return Future< AsyncCompletions >( boost::move( future ) ); -} - - Location ClangCompleter::GetDeclarationLocation( std::string filename, int line, @@ -283,216 +166,8 @@ Location ClangCompleter::GetDefinitionLocation( } -void ClangCompleter::DeleteCachesForFileAsync( std::string filename ) { - file_cache_delete_stack_.Push( filename ); -} - - -void ClangCompleter::DeleteCaches() { - std::vector< std::string > filenames; - - if ( !file_cache_delete_stack_.PopAllNoWait( filenames ) ) - return; - - foreach( const std::string & filename, filenames ) { - translation_unit_store_.Remove( filename ); - } -} - - -bool ClangCompleter::ShouldSkipClangResultCache( const std::string &query, - int line, - int column ) { - // We need query.empty() in addition to the second check because if we don't - // have it, then we have a problem in the following situation: - // The user has a variable 'foo' of type 'A' and a variable 'bar' of type 'B'. - // He then types in 'foo.' and we store the clang results and also return - // them. The user then deletes that code and types in 'bar.'. Without the - // query.empty() check we would return the results from the cache here (it's - // the same line and column!) which would be incorrect. - return query.empty() || latest_clang_results_ - .NewPositionDifferentFromStoredPosition( line, column ); -} - - -// Copy-ctor for unique_future is private in C++03 mode so we need to take it as -// an out param -void ClangCompleter::CreateSortingTask( - const std::string &query, - unique_future< AsyncCompletions > &future ) { - // Careful! The code in this function may burn your eyes. - - function< CompletionDatas( const CompletionDatas & ) > - sort_candidates_for_query_functor = - boost::bind( &ClangCompleter::SortCandidatesForQuery, - boost::ref( *this ), - query, - _1 ); - - function< CompletionDatas() > operate_on_completion_data_functor = - boost::bind( &ClangResultsCache::OperateOnCompletionDatas< CompletionDatas >, - boost::cref( latest_clang_results_ ), - boost::move( sort_candidates_for_query_functor ) ); - - shared_ptr< packaged_task< AsyncCompletions > > task = - boost::make_shared< packaged_task< AsyncCompletions > >( - boost::bind( ReturnValueAsShared< std::vector< CompletionData > >, - boost::move( operate_on_completion_data_functor ) ) ); - - future = task->get_future(); - sorting_task_.Set( task ); -} - - -void ClangCompleter::CreateClangTask( - std::string filename, - int line, - int column, - std::vector< UnsavedFile > unsaved_files, - std::vector< std::string > flags ) { - latest_clang_results_.ResetWithNewLineAndColumn( line, column ); - - function< CompletionDatas() > candidates_for_location_functor = - boost::bind( &ClangCompleter::CandidatesForLocationInFile, - boost::ref( *this ), - boost::move( filename ), - line, - column, - boost::move( unsaved_files ), - boost::move( flags ) ); - - shared_ptr< ClangPackagedTask > clang_packaged_task = - make_shared< ClangPackagedTask >(); - - clang_packaged_task->completions_task_ = - packaged_task< AsyncCompletions >( - boost::bind( ReturnValueAsShared< std::vector< CompletionData > >, - boost::move( candidates_for_location_functor ) ) ); - - clang_task_.Set( clang_packaged_task ); -} - - - -std::vector< CompletionData > ClangCompleter::SortCandidatesForQuery( - const std::string &query, - const std::vector< CompletionData > &completion_datas ) { - Bitset query_bitset = LetterBitsetFromString( query ); - bool query_has_uppercase_letters = any_of( query, is_upper() ); - - std::vector< const Candidate * > repository_candidates = - candidate_repository_.GetCandidatesForStrings( completion_datas ); - - std::vector< ResultAnd< CompletionData * > > data_and_results; - - for ( uint i = 0; i < repository_candidates.size(); ++i ) { - const Candidate *candidate = repository_candidates[ i ]; - - if ( !candidate->MatchesQueryBitset( query_bitset ) ) - continue; - - Result result = candidate->QueryMatchResult( query, - query_has_uppercase_letters ); - - if ( result.IsSubsequence() ) { - ResultAnd< CompletionData * > data_and_result( &completion_datas[ i ], - result ); - data_and_results.push_back( boost::move( data_and_result ) ); - } - } - - std::sort( data_and_results.begin(), data_and_results.end() ); - - std::vector< CompletionData > sorted_completion_datas; - sorted_completion_datas.reserve( data_and_results.size() ); - - foreach ( const ResultAnd< CompletionData * > &data_and_result, - data_and_results ) { - sorted_completion_datas.push_back( *data_and_result.extra_object_ ); - } - - return sorted_completion_datas; -} - - -void ClangCompleter::InitThreads() { - int threads_to_create = - std::max( MIN_ASYNC_THREADS, - std::min( MAX_ASYNC_THREADS, thread::hardware_concurrency() ) ); - - for ( int i = 0; i < threads_to_create; ++i ) { - sorting_threads_.create_thread( bind( &ClangCompleter::SortingThreadMain, - boost::ref( *this ) ) ); - } - - clang_thread_.reset( new thread( &ClangCompleter::ClangThreadMain, - boost::ref( *this ) ) ); -} - - -void ClangCompleter::ClangThreadMain() { - while ( true ) { - try { - shared_ptr< ClangPackagedTask > task = clang_task_.Get(); - - bool has_completions_task = task->completions_task_.valid(); - - if ( has_completions_task ) - task->completions_task_(); - else - task->parsing_task_(); - - if ( !has_completions_task ) { - DeleteCaches(); - continue; - } - - unique_future< AsyncCompletions > future = - task->completions_task_.get_future(); - latest_clang_results_.SetCompletionDatas( *future.get() ); - - { - lock_guard< mutex > lock( clang_data_ready_mutex_ ); - clang_data_ready_ = true; - } - - clang_data_ready_condition_variable_.notify_all(); - } catch ( thread_interrupted & ) { - shared_lock< shared_mutex > lock( time_to_die_mutex_ ); - - if ( time_to_die_ ) - return; - - // else do nothing and re-enter the loop - } - } -} - - -void ClangCompleter::SortingThreadMain() { - while ( true ) { - try { - { - unique_lock< mutex > lock( clang_data_ready_mutex_ ); - - while ( !clang_data_ready_ ) { - clang_data_ready_condition_variable_.wait( lock ); - } - } - - shared_ptr< packaged_task< AsyncCompletions > > task = - sorting_task_.Get(); - - ( *task )(); - } catch ( thread_interrupted & ) { - shared_lock< shared_mutex > lock( time_to_die_mutex_ ); - - if ( time_to_die_ ) - return; - - // else do nothing and re-enter the loop - } - } +void ClangCompleter::DeleteCachesForFile( std::string filename ) { + translation_unit_store_.Remove( filename ); } diff --git a/cpp/ycm/ClangCompleter/ClangCompleter.h b/cpp/ycm/ClangCompleter/ClangCompleter.h index 6ae7819c..46281a4c 100644 --- a/cpp/ycm/ClangCompleter/ClangCompleter.h +++ b/cpp/ycm/ClangCompleter/ClangCompleter.h @@ -23,13 +23,9 @@ #include "Future.h" #include "UnsavedFile.h" #include "Diagnostic.h" -#include "ClangResultsCache.h" #include "TranslationUnitStore.h" #include -#include -#include -#include #include @@ -37,19 +33,12 @@ typedef struct CXTranslationUnitImpl *CXTranslationUnit; namespace YouCompleteMe { -class CandidateRepository; class TranslationUnit; struct CompletionData; struct Location; typedef std::vector< CompletionData > CompletionDatas; -typedef boost::shared_ptr< CompletionDatas > AsyncCompletions; - -typedef boost::unordered_map < std::string, - boost::shared_ptr < - std::vector< std::string > > > FlagsForFile; - // TODO: document that all filename parameters must be absolute paths class ClangCompleter : boost::noncopyable { @@ -57,8 +46,6 @@ public: ClangCompleter(); ~ClangCompleter(); - void EnableThreading(); - std::vector< Diagnostic > DiagnosticsForFile( std::string filename ); bool UpdatingTranslationUnit( const std::string &filename ); @@ -68,20 +55,11 @@ public: // need to ensure we own the memory. // TODO: Change some of these params back to const ref where possible after we // get the server up. - // TODO: Remove the async methods and the threads when the server is ready. - - // Public because of unit tests (gtest is not very thread-friendly) void UpdateTranslationUnit( std::string filename, std::vector< UnsavedFile > unsaved_files, std::vector< std::string > flags ); - Future< void > UpdateTranslationUnitAsync( - std::string filename, - std::vector< UnsavedFile > unsaved_files, - std::vector< std::string > flags ); - - // Public because of unit tests (gtest is not very thread-friendly) std::vector< CompletionData > CandidatesForLocationInFile( std::string filename, int line, @@ -89,14 +67,6 @@ public: std::vector< UnsavedFile > unsaved_files, std::vector< std::string > flags ); - Future< AsyncCompletions > CandidatesForQueryAndLocationInFileAsync( - std::string query, - std::string filename, - int line, - int column, - std::vector< UnsavedFile > unsaved_files, - std::vector< std::string > flags ); - Location GetDeclarationLocation( std::string filename, int line, @@ -111,55 +81,10 @@ public: std::vector< UnsavedFile > unsaved_files, std::vector< std::string > flags ); - void DeleteCachesForFileAsync( std::string filename ); + void DeleteCachesForFile( std::string filename ); private: - void DeleteCaches(); - - // This is basically a union. Only one of the two tasks is set to something - // valid, the other task is invalid. Which one is valid depends on the caller. - struct ClangPackagedTask { - boost::packaged_task< AsyncCompletions > completions_task_; - boost::packaged_task< void > parsing_task_; - }; - - typedef ConcurrentLatestValue < - boost::shared_ptr < - boost::packaged_task< AsyncCompletions > > > LatestSortingTask; - - typedef ConcurrentLatestValue < - boost::shared_ptr< ClangPackagedTask > > LatestClangTask; - - typedef ConcurrentStack< std::string > FileCacheDeleteStack; - - bool ShouldSkipClangResultCache( const std::string &query, - int line, - int column ); - - void CreateSortingTask( const std::string &query, - boost::unique_future< AsyncCompletions > &future ); - - // NOTE: params are taken by value on purpose! With a C++11 compiler we can - // avoid internal copies if params are taken by value (move ctors FTW) - void CreateClangTask( - std::string filename, - int line, - int column, - std::vector< UnsavedFile > unsaved_files, - std::vector< std::string > flags ); - - std::vector< CompletionData > SortCandidatesForQuery( - const std::string &query, - const std::vector< CompletionData > &completion_datas ); - - void InitThreads(); - - void ClangThreadMain(); - - void SortingThreadMain(); - - ///////////////////////////// // PRIVATE MEMBER VARIABLES ///////////////////////////// @@ -167,35 +92,6 @@ private: CXIndex clang_index_; TranslationUnitStore translation_unit_store_; - - CandidateRepository &candidate_repository_; - - bool threading_enabled_; - - // TODO: use boost.atomic for time_to_die_ - bool time_to_die_; - boost::shared_mutex time_to_die_mutex_; - - // TODO: use boost.atomic for clang_data_ready_ - bool clang_data_ready_; - boost::mutex clang_data_ready_mutex_; - boost::condition_variable clang_data_ready_condition_variable_; - - ClangResultsCache latest_clang_results_; - - FileCacheDeleteStack file_cache_delete_stack_; - - // Unfortunately clang is not thread-safe so we need to be careful when we - // access it. Only one thread at a time is allowed to access any single - // translation unit. Currently we only use one thread to access clang and that - // is the thread represented by clang_thread_. - boost::scoped_ptr< boost::thread > clang_thread_; - - boost::thread_group sorting_threads_; - - mutable LatestClangTask clang_task_; - - mutable LatestSortingTask sorting_task_; }; } // namespace YouCompleteMe diff --git a/cpp/ycm/ClangCompleter/ClangResultsCache.cpp b/cpp/ycm/ClangCompleter/ClangResultsCache.cpp deleted file mode 100644 index e11856e3..00000000 --- a/cpp/ycm/ClangCompleter/ClangResultsCache.cpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2011, 2012 Strahinja Val Markovic -// -// This file is part of YouCompleteMe. -// -// YouCompleteMe is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// YouCompleteMe is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with YouCompleteMe. If not, see . - -#include "ClangResultsCache.h" -#include "standard.h" - -using boost::shared_mutex; -using boost::shared_lock; -using boost::unique_lock; - -namespace YouCompleteMe { - -bool ClangResultsCache::NewPositionDifferentFromStoredPosition( int new_line, - int new_colum ) -const { - shared_lock< shared_mutex > reader_lock( access_mutex_ ); - return line_ != new_line || column_ != new_colum; -} - -void ClangResultsCache::ResetWithNewLineAndColumn( int new_line, - int new_colum ) { - unique_lock< shared_mutex > reader_lock( access_mutex_ ); - - line_ = new_line; - column_ = new_colum; - completion_datas_.clear(); -} - -} // namespace YouCompleteMe diff --git a/cpp/ycm/ClangCompleter/ClangResultsCache.h b/cpp/ycm/ClangCompleter/ClangResultsCache.h deleted file mode 100644 index cae4f5fc..00000000 --- a/cpp/ycm/ClangCompleter/ClangResultsCache.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (C) 2011, 2012 Strahinja Val Markovic -// -// This file is part of YouCompleteMe. -// -// YouCompleteMe is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// YouCompleteMe is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with YouCompleteMe. If not, see . - -#ifndef CLANGRESULTSCACHE_H_REUWM3RU -#define CLANGRESULTSCACHE_H_REUWM3RU - -#include "CompletionData.h" - -#include -#include -#include -#include -#include -#include - -namespace YouCompleteMe { - -struct CompletionData; - -class ClangResultsCache : boost::noncopyable { -public: - - ClangResultsCache() : line_( -1 ), column_( -1 ) {} - - bool NewPositionDifferentFromStoredPosition( int new_line, int new_colum ) - const; - - void ResetWithNewLineAndColumn( int new_line, int new_colum ); - - void SetCompletionDatas( - const std::vector< CompletionData > new_completions ) { - completion_datas_ = new_completions; - } - -#ifndef BOOST_NO_RVALUE_REFERENCES -# ifdef __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wc++98-compat" -# endif //#ifdef __clang__ - - void SetCompletionDatas( std::vector< CompletionData > &&new_completions ) { - completion_datas_ = new_completions; - } - -# ifdef __clang__ -# pragma clang diagnostic pop -# endif //#ifdef __clang__ -#endif //#ifndef BOOST_NO_RVALUE_REFERENCES - - template< typename T > - T OperateOnCompletionDatas( - boost::function< T( const std::vector< CompletionData >& ) > operation ) - const { - boost::shared_lock< boost::shared_mutex > reader_lock( access_mutex_ ); - return operation( completion_datas_ ); - } - -private: - - int line_; - int column_; - std::vector< CompletionData > completion_datas_; - - mutable boost::shared_mutex access_mutex_; -}; - -} // namespace YouCompleteMe - -#endif /* end of include guard: CLANGRESULTSCACHE_H_REUWM3RU */ - diff --git a/cpp/ycm/ClangCompleter/TranslationUnitStore.cpp b/cpp/ycm/ClangCompleter/TranslationUnitStore.cpp index 26320024..7c5c0b9e 100644 --- a/cpp/ycm/ClangCompleter/TranslationUnitStore.cpp +++ b/cpp/ycm/ClangCompleter/TranslationUnitStore.cpp @@ -38,14 +38,17 @@ std::size_t HashForFlags( const std::vector< std::string > &flags ) { } // unnamed namespace + TranslationUnitStore::TranslationUnitStore( CXIndex clang_index ) : clang_index_( clang_index ) { } + TranslationUnitStore::~TranslationUnitStore() { RemoveAll(); } + shared_ptr< TranslationUnit > TranslationUnitStore::GetOrCreate( const std::string &filename, const std::vector< UnsavedFile > &unsaved_files, @@ -104,24 +107,28 @@ shared_ptr< TranslationUnit > TranslationUnitStore::GetOrCreate( return unit; } + shared_ptr< TranslationUnit > TranslationUnitStore::Get( const std::string &filename ) { lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ ); return GetNoLock( filename ); } + bool TranslationUnitStore::Remove( const std::string &filename ) { lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ ); Erase( filename_to_flags_hash_, filename ); return Erase( filename_to_translation_unit_, filename ); } + void TranslationUnitStore::RemoveAll() { lock_guard< mutex > lock( filename_to_translation_unit_and_flags_mutex_ ); filename_to_translation_unit_.clear(); filename_to_flags_hash_.clear(); } + shared_ptr< TranslationUnit > TranslationUnitStore::GetNoLock( const std::string &filename ) { return FindWithDefault( filename_to_translation_unit_, diff --git a/cpp/ycm/IdentifierCompleter.cpp b/cpp/ycm/IdentifierCompleter.cpp index 88294ae1..36ca2436 100644 --- a/cpp/ycm/IdentifierCompleter.cpp +++ b/cpp/ycm/IdentifierCompleter.cpp @@ -23,62 +23,16 @@ #include "Result.h" #include "Utils.h" -#include -#include #include -using boost::packaged_task; -using boost::unique_future; -using boost::shared_ptr; -using boost::thread; - namespace YouCompleteMe { -typedef boost::function< std::vector< std::string >() > -FunctionReturnsStringVector; - -extern const unsigned int MAX_ASYNC_THREADS = 4; -extern const unsigned int MIN_ASYNC_THREADS = 2; - -namespace { - -void QueryThreadMain( - IdentifierCompleter::LatestQueryTask &latest_query_task ) { - while ( true ) { - try { - ( *latest_query_task.Get() )(); - } catch ( boost::thread_interrupted & ) { - return; - } - } - -} - -void BufferIdentifiersThreadMain( - IdentifierCompleter::BufferIdentifiersTaskStack - &buffer_identifiers_task_stack ) { - while ( true ) { - try { - ( *buffer_identifiers_task_stack.Pop() )(); - } catch ( boost::thread_interrupted & ) { - return; - } - } -} - - -} // unnamed namespace - - -IdentifierCompleter::IdentifierCompleter() - : threading_enabled_( false ) { -} +IdentifierCompleter::IdentifierCompleter() {} IdentifierCompleter::IdentifierCompleter( - const std::vector< std::string > &candidates ) - : threading_enabled_( false ) { + const std::vector< std::string > &candidates ) { identifier_database_.AddIdentifiers( candidates, "", "" ); } @@ -86,31 +40,11 @@ IdentifierCompleter::IdentifierCompleter( IdentifierCompleter::IdentifierCompleter( const std::vector< std::string > &candidates, const std::string &filetype, - const std::string &filepath ) - : threading_enabled_( false ) { + const std::string &filepath ) { identifier_database_.AddIdentifiers( candidates, filetype, filepath ); } -IdentifierCompleter::~IdentifierCompleter() { - query_threads_.interrupt_all(); - query_threads_.join_all(); - - if ( buffer_identifiers_thread_ ) { - buffer_identifiers_thread_->interrupt(); - buffer_identifiers_thread_->join(); - } -} - - -// We need this mostly so that we can not use it in tests. Apparently the -// GoogleTest framework goes apeshit on us if we enable threads by default. -void IdentifierCompleter::EnableThreading() { - threading_enabled_ = true; - InitThreads(); -} - - void IdentifierCompleter::AddIdentifiersToDatabase( const std::vector< std::string > &new_candidates, const std::string &filetype, @@ -130,22 +64,6 @@ void IdentifierCompleter::AddIdentifiersToDatabaseFromTagFiles( } -void IdentifierCompleter::AddIdentifiersToDatabaseFromTagFilesAsync( - std::vector< std::string > absolute_paths_to_tag_files ) { - // TODO: throw exception when threading is not enabled and this is called - if ( !threading_enabled_ ) - return; - - boost::function< void() > functor = - boost::bind( &IdentifierCompleter::AddIdentifiersToDatabaseFromTagFiles, - boost::ref( *this ), - boost::move( absolute_paths_to_tag_files ) ); - - buffer_identifiers_task_stack_.Push( - boost::make_shared< packaged_task< void > >( boost::move( functor ) ) ); -} - - void IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer( const std::string &buffer_contents, const std::string &filetype, @@ -165,28 +83,6 @@ void IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer( } -void IdentifierCompleter::AddIdentifiersToDatabaseFromBufferAsync( - std::string buffer_contents, - std::string filetype, - std::string filepath, - bool collect_from_comments_and_strings ) { - // TODO: throw exception when threading is not enabled and this is called - if ( !threading_enabled_ ) - return; - - boost::function< void() > functor = - boost::bind( &IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer, - boost::ref( *this ), - boost::move( buffer_contents ), - boost::move( filetype ), - boost::move( filepath ), - collect_from_comments_and_strings ); - - buffer_identifiers_task_stack_.Push( - boost::make_shared< packaged_task< void > >( boost::move( functor ) ) ); -} - - std::vector< std::string > IdentifierCompleter::CandidatesForQuery( const std::string &query ) const { return CandidatesForQueryAndType( query, "" ); @@ -209,46 +105,4 @@ std::vector< std::string > IdentifierCompleter::CandidatesForQueryAndType( } -Future< AsyncResults > IdentifierCompleter::CandidatesForQueryAndTypeAsync( - const std::string &query, - const std::string &filetype ) const { - // TODO: throw exception when threading is not enabled and this is called - if ( !threading_enabled_ ) - return Future< AsyncResults >(); - - FunctionReturnsStringVector functor = - boost::bind( &IdentifierCompleter::CandidatesForQueryAndType, - boost::cref( *this ), - query, - filetype ); - - QueryTask task = - boost::make_shared< packaged_task< AsyncResults > >( - boost::bind( ReturnValueAsShared< std::vector< std::string > >, - boost::move( functor ) ) ); - - unique_future< AsyncResults > future = task->get_future(); - - latest_query_task_.Set( task ); - return Future< AsyncResults >( boost::move( future ) ); -} - - -void IdentifierCompleter::InitThreads() { - int query_threads_to_create = - std::max( MIN_ASYNC_THREADS, - std::min( MAX_ASYNC_THREADS, thread::hardware_concurrency() ) ); - - for ( int i = 0; i < query_threads_to_create; ++i ) { - query_threads_.create_thread( - boost::bind( QueryThreadMain, - boost::ref( latest_query_task_ ) ) ); - } - - buffer_identifiers_thread_.reset( - new boost::thread( BufferIdentifiersThreadMain, - boost::ref( buffer_identifiers_task_stack_ ) ) ); -} - - } // namespace YouCompleteMe diff --git a/cpp/ycm/IdentifierCompleter.h b/cpp/ycm/IdentifierCompleter.h index bc7dd5fd..7eb1edcf 100644 --- a/cpp/ycm/IdentifierCompleter.h +++ b/cpp/ycm/IdentifierCompleter.h @@ -36,8 +36,6 @@ namespace YouCompleteMe { class Candidate; -typedef boost::shared_ptr< std::vector< std::string > > AsyncResults; - class IdentifierCompleter : boost::noncopyable { public: @@ -47,10 +45,6 @@ public: const std::string &filetype, const std::string &filepath ); - ~IdentifierCompleter(); - - void EnableThreading(); - void AddIdentifiersToDatabase( const std::vector< std::string > &new_candidates, const std::string &filetype, @@ -59,25 +53,12 @@ public: void AddIdentifiersToDatabaseFromTagFiles( const std::vector< std::string > &absolute_paths_to_tag_files ); - // NOTE: params are taken by value on purpose! - void AddIdentifiersToDatabaseFromTagFilesAsync( - std::vector< std::string > absolute_paths_to_tag_files ); - void AddIdentifiersToDatabaseFromBuffer( const std::string &buffer_contents, const std::string &filetype, const std::string &filepath, bool collect_from_comments_and_strings ); - // NOTE: params are taken by value on purpose! With a C++11 compiler we can - // avoid an expensive copy of buffer_contents if the param is taken by value - // (move ctors FTW) - void AddIdentifiersToDatabaseFromBufferAsync( - std::string buffer_contents, - std::string filetype, - std::string filepath, - bool collect_from_comments_and_strings ); - // Only provided for tests! std::vector< std::string > CandidatesForQuery( const std::string &query ) const; @@ -86,37 +67,13 @@ public: const std::string &query, const std::string &filetype ) const; - Future< AsyncResults > CandidatesForQueryAndTypeAsync( - const std::string &query, - const std::string &filetype ) const; - - typedef boost::shared_ptr < - boost::packaged_task< AsyncResults > > QueryTask; - - typedef ConcurrentLatestValue< QueryTask > LatestQueryTask; - - typedef ConcurrentStack< VoidTask > BufferIdentifiersTaskStack; - private: - void InitThreads(); - - ///////////////////////////// // PRIVATE MEMBER VARIABLES ///////////////////////////// IdentifierDatabase identifier_database_; - - bool threading_enabled_; - - boost::thread_group query_threads_; - - boost::scoped_ptr< boost::thread > buffer_identifiers_thread_; - - mutable LatestQueryTask latest_query_task_; - - BufferIdentifiersTaskStack buffer_identifiers_task_stack_; }; } // namespace YouCompleteMe diff --git a/cpp/ycm/tests/ClangCompleter/ClangCompleter_test.cpp b/cpp/ycm/tests/ClangCompleter/ClangCompleter_test.cpp index e70f365a..18be50d2 100644 --- a/cpp/ycm/tests/ClangCompleter/ClangCompleter_test.cpp +++ b/cpp/ycm/tests/ClangCompleter/ClangCompleter_test.cpp @@ -43,24 +43,6 @@ TEST( ClangCompleterTest, CandidatesForLocationInFile ) { } -TEST( ClangCompleterTest, CandidatesForQueryAndLocationInFileAsync ) { - ClangCompleter completer; - completer.EnableThreading(); - - Future< AsyncCompletions > completions_future = - completer.CandidatesForQueryAndLocationInFileAsync( - "", - PathToTestFile( "basic.cpp" ).string(), - 11, - 7, - std::vector< UnsavedFile >(), - std::vector< std::string >() ); - - completions_future.Wait(); - - EXPECT_TRUE( !completions_future.GetResults()->empty() ); -} - TEST( ClangCompleterTest, GetDefinitionLocation ) { ClangCompleter completer; std::string filename = PathToTestFile( "basic.cpp" ).string(); diff --git a/cpp/ycm/ycm_core.cpp b/cpp/ycm/ycm_core.cpp index ea353a91..5bb2214f 100644 --- a/cpp/ycm/ycm_core.cpp +++ b/cpp/ycm/ycm_core.cpp @@ -60,36 +60,24 @@ BOOST_PYTHON_MODULE(ycm_core) def( "YcmCoreVersion", YcmCoreVersion ); class_< IdentifierCompleter, boost::noncopyable >( "IdentifierCompleter" ) - .def( "EnableThreading", &IdentifierCompleter::EnableThreading ) .def( "AddIdentifiersToDatabase", &IdentifierCompleter::AddIdentifiersToDatabase ) - .def( "AddIdentifiersToDatabaseFromTagFilesAsync", - &IdentifierCompleter::AddIdentifiersToDatabaseFromTagFilesAsync ) - .def( "AddIdentifiersToDatabaseFromBufferAsync", - &IdentifierCompleter::AddIdentifiersToDatabaseFromBufferAsync ) - .def( "CandidatesForQueryAndTypeAsync", - &IdentifierCompleter::CandidatesForQueryAndTypeAsync ); + .def( "AddIdentifiersToDatabaseFromTagFiles", + &IdentifierCompleter::AddIdentifiersToDatabaseFromTagFiles ) + .def( "AddIdentifiersToDatabaseFromBuffer", + &IdentifierCompleter::AddIdentifiersToDatabaseFromBuffer ) + .def( "CandidatesForQueryAndType", + &IdentifierCompleter::CandidatesForQueryAndType ); // TODO: rename these *Vec classes to *Vector; don't forget the python file class_< std::vector< std::string >, boost::shared_ptr< std::vector< std::string > > >( "StringVec" ) .def( vector_indexing_suite< std::vector< std::string > >() ); - class_< Future< AsyncResults > >( "FutureResults" ) - .def( "ResultsReady", &Future< AsyncResults >::ResultsReady ) - .def( "GetResults", &Future< AsyncResults >::GetResults ); - - class_< Future< void > >( "FutureVoid" ) - .def( "ResultsReady", &Future< void >::ResultsReady ) - .def( "GetResults", &Future< void >::GetResults ); - #ifdef USE_CLANG_COMPLETER def( "ClangVersion", ClangVersion ); - class_< Future< AsyncCompletions > >( "FutureCompletions" ) - .def( "ResultsReady", &Future< AsyncCompletions >::ResultsReady ) - .def( "GetResults", &Future< AsyncCompletions >::GetResults ); - + // TODO: We may not need this at all anymore. Look into it. class_< Future< AsyncCompilationInfoForFile > >( "FutureCompilationInfoForFile" ) .def( "ResultsReady", @@ -116,17 +104,14 @@ BOOST_PYTHON_MODULE(ycm_core) .def( vector_indexing_suite< std::vector< UnsavedFile > >() ); class_< ClangCompleter, boost::noncopyable >( "ClangCompleter" ) - .def( "EnableThreading", &ClangCompleter::EnableThreading ) .def( "DiagnosticsForFile", &ClangCompleter::DiagnosticsForFile ) .def( "GetDeclarationLocation", &ClangCompleter::GetDeclarationLocation ) .def( "GetDefinitionLocation", &ClangCompleter::GetDefinitionLocation ) - .def( "DeleteCachesForFileAsync", - &ClangCompleter::DeleteCachesForFileAsync ) + .def( "DeleteCachesForFile", &ClangCompleter::DeleteCachesForFile ) .def( "UpdatingTranslationUnit", &ClangCompleter::UpdatingTranslationUnit ) - .def( "UpdateTranslationUnitAsync", - &ClangCompleter::UpdateTranslationUnitAsync ) - .def( "CandidatesForQueryAndLocationInFileAsync", - &ClangCompleter::CandidatesForQueryAndLocationInFileAsync ); + .def( "UpdateTranslationUnit", &ClangCompleter::UpdateTranslationUnit ) + .def( "CandidatesForLocationInFile", + &ClangCompleter::CandidatesForLocationInFile ); class_< CompletionData >( "CompletionData" ) .def( "TextToInsertInBuffer", &CompletionData::TextToInsertInBuffer ) diff --git a/python/test_requirements.txt b/python/test_requirements.txt index 65164842..d9d5206d 100644 --- a/python/test_requirements.txt +++ b/python/test_requirements.txt @@ -1,3 +1,5 @@ flake8>=2.0 mock>=1.0.1 nose>=1.3.0 +PyHamcrest>=1.7.2 + diff --git a/python/ycm/completers/all/identifier_completer.py b/python/ycm/completers/all/identifier_completer.py index b1f75db3..c3e27bfb 100644 --- a/python/ycm/completers/all/identifier_completer.py +++ b/python/ycm/completers/all/identifier_completer.py @@ -34,9 +34,8 @@ SYNTAX_FILENAME = 'YCM_PLACEHOLDER_FOR_SYNTAX' class IdentifierCompleter( GeneralCompleter ): def __init__( self, user_options ): super( IdentifierCompleter, self ).__init__( user_options ) - self.completer = ycm_core.IdentifierCompleter() - self.completer.EnableThreading() - self.tags_file_last_mtime = defaultdict( int ) + self._completer = ycm_core.IdentifierCompleter() + self._tags_file_last_mtime = defaultdict( int ) self._logger = logging.getLogger( __name__ ) @@ -44,11 +43,20 @@ class IdentifierCompleter( GeneralCompleter ): return self.QueryLengthAboveMinThreshold( request_data ) - def CandidatesForQueryAsync( self, request_data ): - self.completions_future = self.completer.CandidatesForQueryAndTypeAsync( + def ComputeCandidates( self, request_data ): + if not self.ShouldUseNow( request_data ): + return [] + + completions = self._completer.CandidatesForQueryAndType( ToUtf8IfNeeded( utils.SanitizeQuery( request_data[ 'query' ] ) ), ToUtf8IfNeeded( request_data[ 'filetypes' ][ 0 ] ) ) + completions = completions[ : MAX_IDENTIFIER_COMPLETIONS_RETURNED ] + completions = _RemoveSmallCandidates( + completions, self.user_options[ 'min_num_identifier_candidate_chars' ] ) + + return [ responses.BuildCompletionData( x ) for x in completions ] + def AddIdentifier( self, identifier, request_data ): filetype = request_data[ 'filetypes' ][ 0 ] @@ -60,7 +68,7 @@ class IdentifierCompleter( GeneralCompleter ): vector = ycm_core.StringVec() vector.append( ToUtf8IfNeeded( identifier ) ) self._logger.info( 'Adding ONE buffer identifier for file: %s', filepath ) - self.completer.AddIdentifiersToDatabase( vector, + self._completer.AddIdentifiersToDatabase( vector, ToUtf8IfNeeded( filetype ), ToUtf8IfNeeded( filepath ) ) @@ -92,7 +100,7 @@ class IdentifierCompleter( GeneralCompleter ): text = request_data[ 'file_data' ][ filepath ][ 'contents' ] self._logger.info( 'Adding buffer identifiers for file: %s', filepath ) - self.completer.AddIdentifiersToDatabaseFromBufferAsync( + self._completer.AddIdentifiersToDatabaseFromBuffer( ToUtf8IfNeeded( text ), ToUtf8IfNeeded( filetype ), ToUtf8IfNeeded( filepath ), @@ -106,20 +114,20 @@ class IdentifierCompleter( GeneralCompleter ): current_mtime = os.path.getmtime( tag_file ) except: continue - last_mtime = self.tags_file_last_mtime[ tag_file ] + last_mtime = self._tags_file_last_mtime[ tag_file ] # We don't want to repeatedly process the same file over and over; we only # process if it's changed since the last time we looked at it if current_mtime <= last_mtime: continue - self.tags_file_last_mtime[ tag_file ] = current_mtime + self._tags_file_last_mtime[ tag_file ] = current_mtime absolute_paths_to_tag_files.append( ToUtf8IfNeeded( tag_file ) ) if not absolute_paths_to_tag_files: return - self.completer.AddIdentifiersToDatabaseFromTagFilesAsync( + self._completer.AddIdentifiersToDatabaseFromTagFiles( absolute_paths_to_tag_files ) @@ -129,7 +137,7 @@ class IdentifierCompleter( GeneralCompleter ): keyword_vector.append( ToUtf8IfNeeded( keyword ) ) filepath = SYNTAX_FILENAME + filetypes[ 0 ] - self.completer.AddIdentifiersToDatabase( keyword_vector, + self._completer.AddIdentifiersToDatabase( keyword_vector, ToUtf8IfNeeded( filetypes[ 0 ] ), ToUtf8IfNeeded( filepath ) ) @@ -151,18 +159,6 @@ class IdentifierCompleter( GeneralCompleter ): self.AddPreviousIdentifier( request_data ) - def CandidatesFromStoredRequest( self ): - if not self.completions_future: - return [] - completions = self.completions_future.GetResults()[ - : MAX_IDENTIFIER_COMPLETIONS_RETURNED ] - - completions = _RemoveSmallCandidates( - completions, self.user_options[ 'min_num_identifier_candidate_chars' ] ) - - return [ responses.BuildCompletionData( x ) for x in completions ] - - def _PreviousIdentifier( min_num_completion_start_chars, request_data ): line_num = request_data[ 'line_num' ] column_num = request_data[ 'column_num' ] diff --git a/python/ycm/completers/completer.py b/python/ycm/completers/completer.py index 7bf055f9..c44b5572 100644 --- a/python/ycm/completers/completer.py +++ b/python/ycm/completers/completer.py @@ -58,44 +58,21 @@ class Completer( object ): and will NOT re-query your completer but will in fact provide fuzzy-search on the candidate strings that were stored in the cache. - CandidatesForQueryAsync() is the main entry point when the user types. For + ComputeCandidates() is the main entry point when the user types. For "foo.bar", the user query is "bar" and completions matching this string should - be shown. The job of CandidatesForQueryAsync() is to merely initiate this - request, which will hopefully be processed in a background thread. You may - want to subclass ThreadedCompleter instead of Completer directly. + be shown. It should return the list of candidates. The format of the result + can be a list of strings or a more complicated list of dictionaries. Use + ycm.server.responses.BuildCompletionData to build the detailed response. See + clang_completer.py to see how its used in practice. - AsyncCandidateRequestReady() is the function that is repeatedly polled until - it returns True. If CandidatesForQueryAsync() started a background task of - collecting the required completions, AsyncCandidateRequestReady() would check - the state of that task and return False until it was completed. - - CandidatesFromStoredRequest() should return the list of candidates. This is - what YCM calls after AsyncCandidateRequestReady() returns True. The format of - the result can be a list of strings or a more complicated list of - dictionaries. See ':h complete-items' for the format, and clang_completer.py - to see how its used in practice. + Again, you probably want to override ComputeCandidatesInner(). You also need to implement the SupportedFiletypes() function which should return a list of strings, where the strings are Vim filetypes your completer supports. - clang_completer.py is a good example of a "complicated" completer that - maintains its own internal cache and therefore directly overrides the "main" - functions in the API instead of the *Inner versions. A good example of a - simple completer that does not do this is omni_completer.py. - - If you're confident your completer doesn't need a background task (think - again, you probably do) because you can "certainly" furnish a response in - under 10ms, then you can perform your backend processing in a synchronous - fashion. You may also need to do this because of technical restrictions (much - like omni_completer.py has to do it because accessing Vim internals is not - thread-safe). But even if you're certain, still try to do the processing in a - background thread. Your completer is unlikely to be merged if it does not, - because synchronous processing will block Vim's GUI thread and that's a very, - VERY bad thing (so try not to do it!). Again, you may want to subclass - ThreadedCompleter instead of Completer directly; ThreadedCompleter will - abstract away the use of a background thread for you. See - threaded_completer.py. + clang_completer.py is a good example of a "complicated" completer that A good + example of a simple completer is ultisnips_completer.py. The On* functions are provided for your convenience. They are called when their specific events occur. For instance, the identifier completer collects @@ -122,7 +99,7 @@ class Completer( object ): self.triggers_for_filetype = TriggersForFiletype( user_options[ 'semantic_triggers' ] ) self.completions_future = None - self.completions_cache = None + self._completions_cache = None # It's highly likely you DON'T want to override this function but the *Inner @@ -130,13 +107,13 @@ class Completer( object ): def ShouldUseNow( self, request_data ): inner_says_yes = self.ShouldUseNowInner( request_data ) if not inner_says_yes: - self.completions_cache = None + self._completions_cache = None - previous_results_were_empty = ( self.completions_cache and - self.completions_cache.CacheValid( + previous_results_were_empty = ( self._completions_cache and + self._completions_cache.CacheValid( request_data[ 'line_num' ], request_data[ 'start_column' ] ) and - not self.completions_cache.raw_completions ) + not self._completions_cache.raw_completions ) return inner_says_yes and not previous_results_were_empty @@ -171,20 +148,28 @@ class Completer( object ): # It's highly likely you DON'T want to override this function but the *Inner # version of it. - def CandidatesForQueryAsync( self, request_data ): - self.request_data = request_data + def ComputeCandidates( self, request_data ): + if not self.ShouldUseNow( request_data ): + return [] if ( request_data[ 'query' ] and - self.completions_cache and - self.completions_cache.CacheValid( request_data[ 'line_num' ], - request_data[ 'start_column' ] ) ): - self.completions_cache.filtered_completions = ( - self.FilterAndSortCandidates( - self.completions_cache.raw_completions, - request_data[ 'query' ] ) ) + self._completions_cache and + self._completions_cache.CacheValid( request_data[ 'line_num' ], + request_data[ 'start_column' ] ) ): + return self.FilterAndSortCandidates( + self._completions_cache.raw_completions, + request_data[ 'query' ] ) else: - self.completions_cache = None - self.CandidatesForQueryAsyncInner( request_data ) + self._completions_cache = CompletionsCache() + self._completions_cache.raw_completions = self.ComputeCandidatesInner( + request_data ) + self._completions_cache.line = request_data[ 'line_num' ] + self._completions_cache.column = request_data[ 'start_column' ] + return self._completions_cache.raw_completions + + + def ComputeCandidatesInner( self, request_data ): + pass def DefinedSubcommands( self ): @@ -224,46 +209,6 @@ class Completer( object ): return matches - def CandidatesForQueryAsyncInner( self, query, start_column ): - pass - - - # It's highly likely you DON'T want to override this function but the *Inner - # version of it. - def AsyncCandidateRequestReady( self ): - if self.completions_cache: - return True - else: - return self.AsyncCandidateRequestReadyInner() - - - def AsyncCandidateRequestReadyInner( self ): - if not self.completions_future: - # We return True so that the caller can extract the default value from the - # future - return True - return self.completions_future.ResultsReady() - - - # It's highly likely you DON'T want to override this function but the *Inner - # version of it. - def CandidatesFromStoredRequest( self ): - if self.completions_cache: - return self.completions_cache.filtered_completions - else: - self.completions_cache = CompletionsCache() - self.completions_cache.raw_completions = self.CandidatesFromStoredRequestInner() - self.completions_cache.line = self.request_data[ 'line_num' ] - self.completions_cache.column = self.request_data[ 'start_column' ] - return self.completions_cache.raw_completions - - - def CandidatesFromStoredRequestInner( self ): - if not self.completions_future: - return [] - return self.completions_future.GetResults() - - def OnFileReadyToParse( self, request_data ): pass diff --git a/python/ycm/completers/cpp/clang_completer.py b/python/ycm/completers/cpp/clang_completer.py index 339379ce..31a0bf10 100644 --- a/python/ycm/completers/cpp/clang_completer.py +++ b/python/ycm/completers/cpp/clang_completer.py @@ -30,8 +30,8 @@ CLANG_FILETYPES = set( [ 'c', 'cpp', 'objc', 'objcpp' ] ) MIN_LINES_IN_FILE_TO_PARSE = 5 PARSING_FILE_MESSAGE = 'Still parsing file, no completions yet.' NO_COMPILE_FLAGS_MESSAGE = 'Still no compile flags, no completions yet.' -NO_COMPLETIONS_MESSAGE = 'No completions found; errors in the file?' INVALID_FILE_MESSAGE = 'File is invalid.' +NO_COMPLETIONS_MESSAGE = 'No completions found; errors in the file?' FILE_TOO_SHORT_MESSAGE = ( 'File is less than {} lines long; not compiling.'.format( MIN_LINES_IN_FILE_TO_PARSE ) ) @@ -41,25 +41,14 @@ NO_DIAGNOSTIC_MESSAGE = 'No diagnostic for current line!' class ClangCompleter( Completer ): def __init__( self, user_options ): super( ClangCompleter, self ).__init__( user_options ) - self.max_diagnostics_to_display = user_options[ + self._max_diagnostics_to_display = user_options[ 'max_diagnostics_to_display' ] - self.completer = ycm_core.ClangCompleter() - self.completer.EnableThreading() - self.last_prepared_diagnostics = [] - self.parse_future = None - self.flags = Flags() - self.diagnostic_store = None + self._completer = ycm_core.ClangCompleter() + self._last_prepared_diagnostics = [] + self._flags = Flags() + self._diagnostic_store = None self._logger = logging.getLogger( __name__ ) - # We set this flag when a compilation request comes in while one is already - # in progress. We use this to trigger the pending request after the previous - # one completes (from GetDiagnosticsForCurrentFile because that's the only - # method that knows when the compilation has finished). - # TODO: Remove this now that we have multiple threads in the server; the - # subsequent requests that want to parse will just block until the current - # parse is done and will then proceed. - self.extra_parse_desired = False - def SupportedFiletypes( self ): return CLANG_FILETYPES @@ -84,53 +73,35 @@ class ClangCompleter( Completer ): return files - def CandidatesForQueryAsync( self, request_data ): + def ComputeCandidatesInner( self, request_data ): filename = request_data[ 'filepath' ] - if not filename: return - if self.completer.UpdatingTranslationUnit( ToUtf8IfNeeded( filename ) ): - self.completions_future = None + if self._completer.UpdatingTranslationUnit( ToUtf8IfNeeded( filename ) ): self._logger.info( PARSING_FILE_MESSAGE ) - return responses.BuildDisplayMessageResponse( - PARSING_FILE_MESSAGE ) + raise RuntimeError( PARSING_FILE_MESSAGE ) flags = self._FlagsForRequest( request_data ) if not flags: - self.completions_future = None self._logger.info( NO_COMPILE_FLAGS_MESSAGE ) - return responses.BuildDisplayMessageResponse( - NO_COMPILE_FLAGS_MESSAGE ) - - # TODO: sanitize query, probably in C++ code - - files = ycm_core.UnsavedFileVec() - query = request_data[ 'query' ] - if not query: - files = self.GetUnsavedFilesVector( request_data ) + raise RuntimeError( NO_COMPILE_FLAGS_MESSAGE ) + files = self.GetUnsavedFilesVector( request_data ) line = request_data[ 'line_num' ] + 1 column = request_data[ 'start_column' ] + 1 - self.completions_future = ( - self.completer.CandidatesForQueryAndLocationInFileAsync( - ToUtf8IfNeeded( query ), + results = self._completer.CandidatesForLocationInFile( ToUtf8IfNeeded( filename ), line, column, files, - flags ) ) + flags ) - - def CandidatesFromStoredRequest( self ): - if not self.completions_future: - return [] - results = [ ConvertCompletionData( x ) for x in - self.completions_future.GetResults() ] if not results: self._logger.warning( NO_COMPLETIONS_MESSAGE ) raise RuntimeError( NO_COMPLETIONS_MESSAGE ) - return results + + return [ ConvertCompletionData( x ) for x in results ] def DefinedSubcommands( self ): @@ -170,7 +141,7 @@ class ClangCompleter( Completer ): files = self.GetUnsavedFilesVector( request_data ) line = request_data[ 'line_num' ] + 1 column = request_data[ 'column_num' ] + 1 - return getattr( self.completer, goto_function )( + return getattr( self._completer, goto_function )( ToUtf8IfNeeded( filename ), line, column, @@ -212,73 +183,63 @@ class ClangCompleter( Completer ): def _ClearCompilationFlagCache( self ): - self.flags.Clear() + self._flags.Clear() def OnFileReadyToParse( self, request_data ): filename = request_data[ 'filepath' ] contents = request_data[ 'file_data' ][ filename ][ 'contents' ] if contents.count( '\n' ) < MIN_LINES_IN_FILE_TO_PARSE: - self.parse_future = None self._logger.warning( FILE_TOO_SHORT_MESSAGE ) raise ValueError( FILE_TOO_SHORT_MESSAGE ) if not filename: self._logger.warning( INVALID_FILE_MESSAGE ) - return responses.BuildDisplayMessageResponse( - INVALID_FILE_MESSAGE ) - - if self.completer.UpdatingTranslationUnit( ToUtf8IfNeeded( filename ) ): - self.extra_parse_desired = True - return + raise ValueError( INVALID_FILE_MESSAGE ) flags = self._FlagsForRequest( request_data ) if not flags: - self.parse_future = None self._logger.info( NO_COMPILE_FLAGS_MESSAGE ) - return responses.BuildDisplayMessageResponse( - NO_COMPILE_FLAGS_MESSAGE ) + raise ValueError( NO_COMPILE_FLAGS_MESSAGE ) - self.parse_future = self.completer.UpdateTranslationUnitAsync( + self._completer.UpdateTranslationUnit( ToUtf8IfNeeded( filename ), self.GetUnsavedFilesVector( request_data ), flags ) - self.extra_parse_desired = False - def OnBufferUnload( self, request_data ): - self.completer.DeleteCachesForFileAsync( + self._completer.DeleteCachesForFile( ToUtf8IfNeeded( request_data[ 'unloaded_buffer' ] ) ) def DiagnosticsForCurrentFileReady( self ): - if not self.parse_future: - return False - - return self.parse_future.ResultsReady() + # if not self.parse_future: + # return False + # return self.parse_future.ResultsReady() + pass def GettingCompletions( self, request_data ): - return self.completer.UpdatingTranslationUnit( + return self._completer.UpdatingTranslationUnit( ToUtf8IfNeeded( request_data[ 'filepath' ] ) ) def GetDiagnosticsForCurrentFile( self, request_data ): filename = request_data[ 'filepath' ] if self.DiagnosticsForCurrentFileReady(): - diagnostics = self.completer.DiagnosticsForFile( + diagnostics = self._completer.DiagnosticsForFile( ToUtf8IfNeeded( filename ) ) - self.diagnostic_store = DiagnosticsToDiagStructure( diagnostics ) - self.last_prepared_diagnostics = [ + self._diagnostic_store = DiagnosticsToDiagStructure( diagnostics ) + self._last_prepared_diagnostics = [ responses.BuildDiagnosticData( x ) for x in - diagnostics[ : self.max_diagnostics_to_display ] ] - self.parse_future = None + diagnostics[ : self._max_diagnostics_to_display ] ] + # self.parse_future = None - if self.extra_parse_desired: - self.OnFileReadyToParse( request_data ) + # if self.extra_parse_desired: + # self.OnFileReadyToParse( request_data ) - return self.last_prepared_diagnostics + return self._last_prepared_diagnostics def GetDetailedDiagnostic( self, request_data ): @@ -286,11 +247,11 @@ class ClangCompleter( Completer ): current_column = request_data[ 'column_num' ] + 1 current_file = request_data[ 'filepath' ] - if not self.diagnostic_store: + if not self._diagnostic_store: return responses.BuildDisplayMessageResponse( NO_DIAGNOSTIC_MESSAGE ) - diagnostics = self.diagnostic_store[ current_file ][ current_line ] + diagnostics = self._diagnostic_store[ current_file ][ current_line ] if not diagnostics: return responses.BuildDisplayMessageResponse( NO_DIAGNOSTIC_MESSAGE ) @@ -308,11 +269,6 @@ class ClangCompleter( Completer ): closest_diagnostic.long_formatted_text_ ) - def ShouldUseNow( self, request_data ): - # We don't want to use the Completer API cache, we use one in the C++ code. - return self.ShouldUseNowInner( request_data ) - - def DebugInfo( self, request_data ): filename = request_data[ 'filepath' ] if not filename: @@ -328,7 +284,7 @@ class ClangCompleter( Completer ): if 'compilation_flags' in request_data: return PrepareFlagsForClang( request_data[ 'compilation_flags' ], filename ) - return self.flags.FlagsForFile( filename ) + return self._flags.FlagsForFile( filename ) # TODO: Make this work again # def DiagnosticToDict( diagnostic ): diff --git a/python/ycm/completers/cs/cs_completer.py b/python/ycm/completers/cs/cs_completer.py index 4943ccae..5af69592 100755 --- a/python/ycm/completers/cs/cs_completer.py +++ b/python/ycm/completers/cs/cs_completer.py @@ -21,7 +21,7 @@ import os from sys import platform import glob -from ycm.completers.threaded_completer import ThreadedCompleter +from ycm.completers.completer import Completer from ycm.server import responses from ycm import utils import urllib2 @@ -36,7 +36,7 @@ SERVER_NOT_FOUND_MSG = ( 'OmniSharp server binary not found at {0}. ' + 'Did you compile it? You can do so by running ' + '"./install.sh --omnisharp-completer".' ) -class CsharpCompleter( ThreadedCompleter ): +class CsharpCompleter( Completer ): """ A Completer that uses the Omnisharp server as completion engine. """ @@ -61,7 +61,7 @@ class CsharpCompleter( ThreadedCompleter ): return [ 'cs' ] - def ComputeCandidates( self, request_data ): + def ComputeCandidatesInner( self, request_data ): return [ responses.BuildCompletionData( completion[ 'CompletionText' ], completion[ 'DisplayText' ], diff --git a/python/ycm/completers/general/filename_completer.py b/python/ycm/completers/general/filename_completer.py index 5d9eb0da..b6549b64 100644 --- a/python/ycm/completers/general/filename_completer.py +++ b/python/ycm/completers/general/filename_completer.py @@ -19,12 +19,12 @@ import os import re -from ycm.completers.threaded_completer import ThreadedCompleter +from ycm.completers.completer import Completer from ycm.completers.cpp.clang_completer import InCFamilyFile from ycm.completers.cpp.flags import Flags from ycm.server import responses -class FilenameCompleter( ThreadedCompleter ): +class FilenameCompleter( Completer ): """ General completer that provides filename and filepath completions. """ @@ -73,7 +73,7 @@ class FilenameCompleter( ThreadedCompleter ): return [] - def ComputeCandidates( self, request_data ): + def ComputeCandidatesInner( self, request_data ): current_line = request_data[ 'line_value' ] start_column = request_data[ 'start_column' ] filepath = request_data[ 'filepath' ] diff --git a/python/ycm/completers/general/general_completer_store.py b/python/ycm/completers/general/general_completer_store.py index 850b39ed..f47dff8d 100644 --- a/python/ycm/completers/general/general_completer_store.py +++ b/python/ycm/completers/general/general_completer_store.py @@ -74,20 +74,13 @@ class GeneralCompleterStore( Completer ): return should_use_now - def CandidatesForQueryAsync( self, request_data ): - for completer in self._current_query_completers: - completer.CandidatesForQueryAsync( request_data ) + def ComputeCandidates( self, request_data ): + if not self.ShouldUseNow( request_data ): + return [] - - def AsyncCandidateRequestReady( self ): - return all( x.AsyncCandidateRequestReady() for x in - self._current_query_completers ) - - - def CandidatesFromStoredRequest( self ): candidates = [] for completer in self._current_query_completers: - candidates += completer.CandidatesFromStoredRequest() + candidates += completer.ComputeCandidates( request_data ) return candidates diff --git a/python/ycm/completers/general/ultisnips_completer.py b/python/ycm/completers/general/ultisnips_completer.py index 15858585..7a364f4d 100644 --- a/python/ycm/completers/general/ultisnips_completer.py +++ b/python/ycm/completers/general/ultisnips_completer.py @@ -33,23 +33,17 @@ class UltiSnipsCompleter( GeneralCompleter ): self._filtered_candidates = None - def ShouldUseNowInner( self, request_data ): + def ShouldUseNow( self, request_data ): return self.QueryLengthAboveMinThreshold( request_data ) - def CandidatesForQueryAsync( self, request_data ): - self._filtered_candidates = self.FilterAndSortCandidates( + def ComputeCandidates( self, request_data ): + if not self.ShouldUseNow( request_data ): + return [] + return self.FilterAndSortCandidates( self._candidates, request_data[ 'query' ] ) - def AsyncCandidateRequestReady( self ): - return True - - - def CandidatesFromStoredRequest( self ): - return self._filtered_candidates if self._filtered_candidates else [] - - def OnBufferVisit( self, request_data ): raw_candidates = request_data[ 'ultisnips_snippets' ] self._candidates = [ diff --git a/python/ycm/completers/python/jedi_completer.py b/python/ycm/completers/python/jedi_completer.py index b2149076..f1599276 100644 --- a/python/ycm/completers/python/jedi_completer.py +++ b/python/ycm/completers/python/jedi_completer.py @@ -19,7 +19,7 @@ # You should have received a copy of the GNU General Public License # along with YouCompleteMe. If not, see . -from ycm.completers.threaded_completer import ThreadedCompleter +from ycm.completers.completer import Completer from ycm.server import responses import sys @@ -38,7 +38,7 @@ except ImportError: sys.path.pop( 0 ) -class JediCompleter( ThreadedCompleter ): +class JediCompleter( Completer ): """ A Completer that uses the Jedi completion engine. https://jedi.readthedocs.org/en/latest/ @@ -63,7 +63,7 @@ class JediCompleter( ThreadedCompleter ): return jedi.Script( contents, line, column, filename ) - def ComputeCandidates( self, request_data ): + def ComputeCandidatesInner( self, request_data ): script = self._GetJediScript( request_data ) return [ responses.BuildCompletionData( str( completion.name ), diff --git a/python/ycm/completers/threaded_completer.py b/python/ycm/completers/threaded_completer.py deleted file mode 100644 index cf1a1b13..00000000 --- a/python/ycm/completers/threaded_completer.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2011, 2012 Strahinja Val Markovic -# -# This file is part of YouCompleteMe. -# -# YouCompleteMe is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# YouCompleteMe is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with YouCompleteMe. If not, see . - -import abc -from threading import Thread, Event -from ycm.completers.completer import Completer - -class ThreadedCompleter( Completer ): - """A subclass of Completer that abstracts away the use of a background thread. - - This is a great class to subclass for your custom completer. It will provide - you with async computation of candidates when you implement the - ComputeCandidates() method; no need to worry about threads, locks, events or - similar. - - If you use this class as the base class for your Completer, you DON'T need to - provide implementations for the CandidatesForQueryAsync(), - AsyncCandidateRequestReady() and CandidatesFromStoredRequest() functions. Just - implement ComputeCandidates(). - - For examples of subclasses of this class, see the following files: - python/completers/general/filename_completer.py - """ - - def __init__( self, user_options ): - super( ThreadedCompleter, self ).__init__( user_options ) - self._query_ready = Event() - self._candidates_ready = Event() - self._candidates = None - self._start_completion_thread() - - - def _start_completion_thread( self ): - self._completion_thread = Thread( target=self.SetCandidates ) - self._completion_thread.daemon = True - self._completion_thread.start() - - - def CandidatesForQueryAsyncInner( self, request_data ): - self._candidates = None - self._candidates_ready.clear() - self._request_data = request_data - self._query_ready.set() - - - def AsyncCandidateRequestReadyInner( self ): - return WaitAndClearIfSet( self._candidates_ready, timeout=0.005 ) - - - def CandidatesFromStoredRequestInner( self ): - return self._candidates or [] - - - @abc.abstractmethod - def ComputeCandidates( self, request_data ): - """This function should compute the candidates to show to the user. - The return value should be of the same type as that for - CandidatesFromStoredRequest().""" - pass - - - def SetCandidates( self ): - while True: - try: - WaitAndClearIfSet( self._query_ready ) - self._candidates = self.ComputeCandidates( self._request_data ) - except: - self._query_ready.clear() - self._candidates = [] - self._candidates_ready.set() - - -def WaitAndClearIfSet( event, timeout=None ): - """Given an |event| and a |timeout|, waits for the event a maximum of timeout - seconds. After waiting, clears the event if it's set and returns the state of - the event before it was cleared.""" - - # We can't just do flag_is_set = event.wait( timeout ) because that breaks on - # Python 2.6 - event.wait( timeout ) - flag_is_set = event.is_set() - if flag_is_set: - event.clear() - return flag_is_set diff --git a/python/ycm/server/tests/basic_test.py b/python/ycm/server/tests/basic_test.py index ad8b52a8..74b24e13 100644 --- a/python/ycm/server/tests/basic_test.py +++ b/python/ycm/server/tests/basic_test.py @@ -21,6 +21,7 @@ from webtest import TestApp from .. import ycmd from ..responses import BuildCompletionData from nose.tools import ok_, eq_, with_setup +from hamcrest import assert_that, has_items, has_entry import bottle bottle.debug( True ) @@ -41,6 +42,11 @@ def RequestDataForFileWithContents( filename, contents = None ): } +# TODO: Make the other tests use this helper too instead of BuildCompletionData +def CompletionEntryMatcher( insertion_text ): + return has_entry( 'insertion_text', insertion_text ) + + def Setup(): ycmd.SetServerStateToDefaults() @@ -69,6 +75,48 @@ def GetCompletions_IdentifierCompleter_Works_test(): app.post_json( '/get_completions', completion_data ).json ) +@with_setup( Setup ) +def GetCompletions_ClangCompleter_Works_test(): + app = TestApp( ycmd.app ) + contents = """ +struct Foo { + int x; + int y; + char c; +}; + +int main() +{ + Foo foo; + foo. +} +""" + + filename = '/foo.cpp' + completion_data = { + 'compilation_flags': ['-x', 'c++'], + # 0-based line and column! + 'query': '', + 'line_num': 10, + 'column_num': 6, + 'start_column': 6, + 'line_value': ' foo.', + 'filetypes': ['cpp'], + 'filepath': filename, + 'file_data': { + filename: { + 'contents': contents, + 'filetypes': ['cpp'] + } + } + } + + results = app.post_json( '/get_completions', completion_data ).json + assert_that( results, has_items( CompletionEntryMatcher( 'c' ), + CompletionEntryMatcher( 'x' ), + CompletionEntryMatcher( 'y' ) ) ) + + @with_setup( Setup ) def GetCompletions_IdentifierCompleter_SyntaxKeywordsAdded_test(): app = TestApp( ycmd.app ) diff --git a/python/ycm/server/ycmd.py b/python/ycm/server/ycmd.py index 2730e56f..098e95d1 100755 --- a/python/ycm/server/ycmd.py +++ b/python/ycm/server/ycmd.py @@ -31,7 +31,6 @@ sys.path.insert( 0, os.path.join( '../..' ) ) import logging -import time import json import bottle from bottle import run, request, response @@ -89,16 +88,7 @@ def GetCompletions(): do_filetype_completion else SERVER_STATE.GetGeneralCompleter() ) - # This is necessary so that general_completer_store fills up - # _current_query_completers. - # TODO: Fix this. - completer.ShouldUseNow( request_data ) - - # TODO: This should not be async anymore, server is multi-threaded - completer.CandidatesForQueryAsync( request_data ) - while not completer.AsyncCandidateRequestReady(): - time.sleep( 0.03 ) - return _JsonResponse( completer.CandidatesFromStoredRequest() ) + return _JsonResponse( completer.ComputeCandidates( request_data ) ) @app.get( '/user_options' )