From 7cf580a4472b831243c662700798c239960f2035 Mon Sep 17 00:00:00 2001 From: Strahinja Val Markovic Date: Sat, 5 May 2012 23:48:22 -0700 Subject: [PATCH] Completion suggestions are now fetched async --- autoload/youcompleteme.vim | 24 ++++++--- cpp/CMakeLists.txt | 2 +- cpp/Completer.cpp | 104 ++++++++++++++++++++++++++++++++++--- cpp/Completer.h | 43 +++++++++++---- cpp/ConcurrentStack.h | 66 +++++++++++++++++++++++ cpp/Future.cpp | 55 ++++++++++++++++++++ cpp/Future.h | 47 +++++++++++++++++ cpp/indexer.cpp | 12 ++++- python/ycm.py | 23 +++++--- 9 files changed, 343 insertions(+), 33 deletions(-) create mode 100644 cpp/ConcurrentStack.h create mode 100644 cpp/Future.cpp create mode 100644 cpp/Future.h diff --git a/autoload/youcompleteme.vim b/autoload/youcompleteme.vim index b14f89b7..5d343f3d 100644 --- a/autoload/youcompleteme.vim +++ b/autoload/youcompleteme.vim @@ -102,13 +102,27 @@ function! youcompleteme#Complete(findstart, base) return start_column else let s:old_cursor_text = a:base - let results = [] if strlen( a:base ) < g:ycm_min_num_of_chars_for_completion - return results + return [] endif + py csystem.CandidatesForQueryAsync( vim.eval('a:base') ) + + let l:results_ready = 0 + while !l:results_ready + py << EOF +results_ready = csystem.AsyncCandidateRequestReady() +if results_ready: + vim.command( 'let l:results_ready = 1' ) +EOF + if complete_check() + return { 'words' : [], 'refresh' : 'always'} + endif + endwhile + + let l:results = [] py << EOF -results = csystem.CompletionCandidatesForQuery( vim.eval('a:base') ) +results = csystem.CandidatesFromStoredRequest() if results: vim.command( 'let l:results = ' + str( results ) ) EOF @@ -117,10 +131,8 @@ EOF " keystroke. The problem is still present in vim 7.3.390 but is fixed in " 7.3.475. It's possible that patch 404 was the one that fixed this issue, " but I haven't tested this assumption. - let dict = { 'words' : results } " A bug in vim causes the '.' register to break when we use set this... sigh - let dict.refresh = 'always' - return dict + return { 'words' : l:results, 'refresh' : 'always'} endif endfunction diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2c19ccdf..71fbb256 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -86,7 +86,7 @@ endif() if( CMAKE_COMPILER_IS_GNUCXX OR COMPILER_IS_CLANG ) # We want all warnings, and warnings should be treated as errors - #add_definitions( -Wall -pedantic -Werror ) + # TODO: -Wextra? add_definitions( -Wall -Werror ) endif() diff --git a/cpp/Completer.cpp b/cpp/Completer.cpp index 2af49b53..73f20068 100644 --- a/cpp/Completer.cpp +++ b/cpp/Completer.cpp @@ -19,14 +19,42 @@ #include "Completer.h" #include "Utils.h" +#include +#include +#include + using boost::python::len; using boost::python::extract; +using boost::packaged_task; +using boost::bind; +using boost::unique_future; +using boost::make_shared; +using boost::shared_ptr; +using boost::bind; +using boost::thread; namespace YouCompleteMe { +namespace +{ + +const unsigned int MAX_ASYNC_THREADS = 4; +const unsigned int MIN_ASYNC_THREADS = 2; + +void ThreadMain( TaskStack &task_stack ) +{ + while ( true ) + { + ( *task_stack.Pop() )(); + } +} + +} // unnamed namespace + Completer::Completer( const Pylist &candidates ) + : threading_enabled_( false ) { AddCandidatesToDatabase( candidates, "", "" ); } @@ -34,7 +62,8 @@ Completer::Completer( const Pylist &candidates ) Completer::Completer( const Pylist &candidates, const std::string &filetype, - const std::string &filepath) + const std::string &filepath ) + : threading_enabled_( false ) { AddCandidatesToDatabase( candidates, filetype, filepath ); } @@ -50,6 +79,15 @@ Completer::~Completer() } +// We need this mostly so that we can not use it in tests. Apparently the +// GoogleTest framework goes apeshit on us if we enable threads by default. +void Completer::EnableThreading() +{ + threading_enabled_ = true; + InitThreads(); +} + + void Completer::AddCandidatesToDatabase( const Pylist &new_candidates, const std::string &filetype, const std::string &filepath ) @@ -85,13 +123,57 @@ void Completer::CandidatesForQuery( const std::string &query, void Completer::CandidatesForQueryAndType( const std::string &query, const std::string &filetype, Pylist &candidates ) const +{ + std::vector< Result > results; + ResultsForQueryAndType( query, filetype, results ); + + foreach ( const Result& result, results ) + { + candidates.append( *result.Text() ); + } +} + + +Future Completer::CandidatesForQueryAndTypeAsync( + const std::string &query, + const std::string &filetype ) const +{ + // TODO: throw exception when threading is not enabled and this is called + if (!threading_enabled_) + return Future(); + + // Try not to look at this too hard, it may burn your eyes. + shared_ptr< packaged_task< AsyncResults > > task = + make_shared< packaged_task< AsyncResults > >( + bind( &Completer::ResultsForQueryAndType, + boost::cref( *this ), + query, + filetype ) ); + + unique_future< AsyncResults > future = task->get_future(); + + task_stack_.Push( task ); + return Future( move( future ) ); +} + +AsyncResults Completer::ResultsForQueryAndType( + const std::string &query, + const std::string &filetype ) const +{ + AsyncResults results = boost::make_shared< std::vector< Result > >(); + ResultsForQueryAndType( query, filetype, *results ); + return results; +} + +void Completer::ResultsForQueryAndType( const std::string &query, + const std::string &filetype, + std::vector< Result > &results ) const { FiletypeMap::const_iterator it = filetype_map_.find( filetype ); if ( it == filetype_map_.end() ) return; Bitset query_bitset = LetterBitsetFromString( query ); - std::vector< Result > results; foreach ( const FilepathToCandidates::value_type &path_and_candidates, *it->second ) @@ -108,11 +190,6 @@ void Completer::CandidatesForQueryAndType( const std::string &query, } std::sort( results.begin(), results.end() ); - - foreach ( const Result& result, results ) - { - candidates.append( *result.Text() ); - } } @@ -136,4 +213,17 @@ std::vector< Candidate* >& Completer::GetCandidateVector( } +void Completer::InitThreads() +{ + int threads_to_create = + std::max( MIN_ASYNC_THREADS, + std::min( MAX_ASYNC_THREADS, thread::hardware_concurrency() ) ); + + for ( int i = 0; i < threads_to_create; ++i ) + { + threads_.create_thread( bind( ThreadMain, boost::ref( task_stack_ ) ) ); + } +} + + } // namespace YouCompleteMe diff --git a/cpp/Completer.h b/cpp/Completer.h index c1ecd515..12e8cfe3 100644 --- a/cpp/Completer.h +++ b/cpp/Completer.h @@ -19,6 +19,8 @@ #define COMPLETER_H_7AR4UGXE #include "Candidate.h" +#include "ConcurrentStack.h" +#include "Future.h" #include #include @@ -28,6 +30,7 @@ #include #include + namespace YouCompleteMe { @@ -44,10 +47,12 @@ typedef boost::unordered_map< std::string, typedef boost::unordered_map< std::string, boost::shared_ptr< FilepathToCandidates > > FiletypeMap; +typedef ConcurrentStack< + boost::shared_ptr< + boost::packaged_task< AsyncResults > > > TaskStack; -// TODO: resolve problems with noncopyable -// class Completer : boost::noncopyable -class Completer + +class Completer : boost::noncopyable { public: Completer() {} @@ -57,6 +62,8 @@ public: const std::string &filepath ); ~Completer(); + void EnableThreading(); + void AddCandidatesToDatabase( const Pylist &new_candidates, const std::string &filetype, const std::string &filepath ); @@ -69,27 +76,41 @@ public: const std::string &filetype, Pylist &candidates ) const; + Future CandidatesForQueryAndTypeAsync( const std::string &query, + const std::string &filetype ) const; + private: + AsyncResults ResultsForQueryAndType( const std::string &query, + const std::string &filetype ) const; + + void ResultsForQueryAndType( const std::string &query, + const std::string &filetype, + std::vector< Result > &results ) const; + std::vector< Candidate* >& GetCandidateVector( const std::string &filetype, const std::string &filepath ); - struct CandidatePointerLess - { - bool operator() ( const Candidate *first, const Candidate *second ) - { - return first->Text() < second->Text(); - } - }; + void InitThreads(); + + + ///////////////////////////// + // PRIVATE MEMBER VARIABLES + ///////////////////////////// // This data structure owns all the Candidate pointers CandidateRepository candidate_repository_; FiletypeMap filetype_map_; + + mutable TaskStack task_stack_; + + bool threading_enabled_; + + boost::thread_group threads_; }; } // namespace YouCompleteMe #endif /* end of include guard: COMPLETER_H_7AR4UGXE */ - diff --git a/cpp/ConcurrentStack.h b/cpp/ConcurrentStack.h new file mode 100644 index 00000000..6198f1d8 --- /dev/null +++ b/cpp/ConcurrentStack.h @@ -0,0 +1,66 @@ +// Copyright (C) 2011, 2012 Strahinja Val Markovic +// +// This file is part of YouCompleteMe. +// +// YouCompleteMe is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// YouCompleteMe is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with YouCompleteMe. If not, see . + +#ifndef CONCURRENTSTACK_H_SYF1JPPG +#define CONCURRENTSTACK_H_SYF1JPPG + +#include +#include +#include + +namespace YouCompleteMe +{ + +template +class ConcurrentStack : boost::noncopyable +{ +public: + + void Push( const T& data ) + { + { + boost::unique_lock< boost::mutex > lock( mutex_ ); + stack_.push( data ); + } + + condition_variable_.notify_one(); + } + + T Pop() + { + boost::unique_lock< boost::mutex > lock( mutex_ ); + + while ( stack_.empty() ) + { + condition_variable_.wait( lock ); + } + + T result = stack_.top(); + stack_.pop(); + return result; + } + +private: + std::stack stack_; + boost::mutex mutex_; + boost::condition_variable condition_variable_; + +}; + +} // namespace YouCompleteMe + +#endif /* end of include guard: CONCURRENTSTACK_H_SYF1JPPG */ diff --git a/cpp/Future.cpp b/cpp/Future.cpp new file mode 100644 index 00000000..72920f20 --- /dev/null +++ b/cpp/Future.cpp @@ -0,0 +1,55 @@ +// Copyright (C) 2011, 2012 Strahinja Val Markovic +// +// This file is part of YouCompleteMe. +// +// YouCompleteMe is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// YouCompleteMe is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with YouCompleteMe. If not, see . + +#include "standard.h" +#include "Future.h" +#include "Result.h" + +namespace YouCompleteMe +{ + +Future::Future( boost::shared_future< AsyncResults > future ) + : future_( boost::move( future ) ) +{ +} + +bool Future::ResultsReady() +{ + return future_.is_ready(); +} + +void Future::GetResults( Pylist &candidates ) +{ + AsyncResults results; + + try + { + results = future_.get(); + } + + catch ( boost::future_uninitialized & ) + { + return; + } + + foreach ( const Result& result, *results ) + { + candidates.append( *result.Text() ); + } +} + +} // namespace YouCompleteMe diff --git a/cpp/Future.h b/cpp/Future.h new file mode 100644 index 00000000..cecfa1c4 --- /dev/null +++ b/cpp/Future.h @@ -0,0 +1,47 @@ +// Copyright (C) 2011, 2012 Strahinja Val Markovic +// +// This file is part of YouCompleteMe. +// +// YouCompleteMe is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// YouCompleteMe is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with YouCompleteMe. If not, see . + +#ifndef FUTURE_H_NR1U6MZS +#define FUTURE_H_NR1U6MZS + +#include +#include +#include + +namespace YouCompleteMe +{ + +class Result; + +typedef boost::python::list Pylist; +typedef boost::shared_ptr< std::vector< Result > > AsyncResults; + +class Future +{ +public: + Future() {} + Future( boost::shared_future< AsyncResults > future ); + bool ResultsReady(); + void GetResults( Pylist &candidates ); + +private: + boost::shared_future< AsyncResults > future_; +}; + +} // namespace YouCompleteMe + +#endif /* end of include guard: FUTURE_H_NR1U6MZS */ diff --git a/cpp/indexer.cpp b/cpp/indexer.cpp index 52918049..372bd9e9 100644 --- a/cpp/indexer.cpp +++ b/cpp/indexer.cpp @@ -16,15 +16,23 @@ // along with YouCompleteMe. If not, see . #include "Completer.h" +#include "Future.h" #include +#include BOOST_PYTHON_MODULE(indexer) { using namespace boost::python; using namespace YouCompleteMe; - class_( "Completer" ) + class_< Future >( "Future" ) + .def( "ResultsReady", &Future::ResultsReady ) + .def( "GetResults", &Future::GetResults ); + + class_< Completer, boost::noncopyable >( "Completer" ) + .def( "EnableThreading", &Completer::EnableThreading ) .def( "AddCandidatesToDatabase", &Completer::AddCandidatesToDatabase ) - .def( "CandidatesForQueryAndType", &Completer::CandidatesForQueryAndType ); + .def( "CandidatesForQueryAndTypeAsync", + &Completer::CandidatesForQueryAndTypeAsync ); } diff --git a/python/ycm.py b/python/ycm.py index eef0e9ed..6747a637 100644 --- a/python/ycm.py +++ b/python/ycm.py @@ -26,15 +26,26 @@ min_num_chars = int( vim.eval( "g:ycm_min_num_of_chars_for_completion" ) ) class CompletionSystem( object ): def __init__( self ): self.completer = indexer.Completer() + self.completer.EnableThreading() self.pattern = re.compile( r"[_a-zA-Z]\w*" ) + self.future = None - def CompletionCandidatesForQuery( self, query ): - candidates = [] + def CandidatesForQueryAsync( self, query ): filetype = vim.eval( "&filetype" ) - self.completer.CandidatesForQueryAndType( SanitizeQuery( query ), - filetype, - candidates ) - return candidates + self.future = self.completer.CandidatesForQueryAndTypeAsync( + SanitizeQuery( query ), + filetype ) + + def AsyncCandidateRequestReady( self ): + return self.future.ResultsReady() + + def CandidatesFromStoredRequest( self ): + if not self.future: + return [] + + results = [] + self.future.GetResults( results ) + return results def AddBufferIdentifiers( self ): text = "\n".join( vim.current.buffer )