vim/YouCompleteMe
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Buffer identifiers are now extracted async

Browse Source
This commit is contained in:
Strahinja Val Markovic 2012-07-24 20:09:09 -07:00
parent 6d76563e86
commit 13f87c5a62
8 changed files with 151 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
cpp/ycm/ConcurrentStack.h Normal file
View File

@ -0,0 +1,66 @@
// Copyright (C) 2011, 2012 Strahinja Val Markovic <val@markovic.io>
//
// This file is part of YouCompleteMe.
//
// YouCompleteMe is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// YouCompleteMe is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with YouCompleteMe. If not, see <http://www.gnu.org/licenses/>.
#ifndef CONCURRENTSTACK_H_TGI0GOR6
#define CONCURRENTSTACK_H_TGI0GOR6
#include <boost/thread.hpp>
#include <boost/utility.hpp>
#include <stack>
namespace YouCompleteMe
{
template <typename T>
class ConcurrentStack : boost::noncopyable
{
public:
void Push( const T& data )
{
{
boost::unique_lock< boost::mutex > lock( mutex_ );
stack_.push( data );
}
condition_variable_.notify_one();
}
T Pop()
{
boost::unique_lock< boost::mutex > lock( mutex_ );
while ( stack_.empty() )
{
condition_variable_.wait( lock );
}
T top = stack_.top();
stack_.pop();
return top;
}
private:
std::stack< T > stack_;
boost::mutex mutex_;
boost::condition_variable condition_variable_;
};
} // namespace YouCompleteMe
#endif /* end of include guard: CONCURRENTSTACK_H_TGI0GOR6 */

56
cpp/ycm/IdentifierCompleter.cpp
View File

@ -44,11 +44,21 @@ extern const unsigned int MIN_ASYNC_THREADS = 2;
namespace
{
void ThreadMain( LatestTask &latest_task )
void QueryThreadMain( LatestQueryTask &latest_query_task )
{
while ( true )
{
( *latest_task.Get() )();
( *latest_query_task.Get() )();
}
}
void BufferIdentifiersThreadMain(
BufferIdentifiersTaskStack &buffer_identifiers_task_stack )
{
while ( true )
{
( *buffer_identifiers_task_stack.Pop() )();
}
}
@ -123,11 +133,20 @@ void IdentifierCompleter::AddCandidatesToDatabaseFromBuffer(
}
void IdentifierCompleter::ClearCandidatesStoredForFile(
const std::string &filetype,
const std::string &filepath )
void IdentifierCompleter::AddCandidatesToDatabaseFromBufferAsync(
std::string buffer_contents,
std::string filetype,
std::string filepath )
{
GetCandidateList( filetype, filepath ).clear();
boost::function< void() > functor =
bind( &IdentifierCompleter::AddCandidatesToDatabaseFromBuffer,
boost::ref( *this ),
boost::move( buffer_contents ),
boost::move( filetype ),
boost::move( filepath ) );
buffer_identifiers_task_stack_.Push(
make_shared< packaged_task< void > >( functor ) );
}
@ -170,15 +189,13 @@ Future< AsyncResults > IdentifierCompleter::CandidatesForQueryAndTypeAsync(
query,
filetype );
// Try not to look at this too hard, it may burn your eyes.
shared_ptr< packaged_task< AsyncResults > > task =
make_shared< packaged_task< AsyncResults > >(
QueryTask task = make_shared< packaged_task< AsyncResults > >(
bind( ReturnValueAsShared< std::vector< std::string > >,
functor ) );
unique_future< AsyncResults > future = task->get_future();
latest_task_.Set( task );
latest_query_task_.Set( task );
return Future< AsyncResults >( boost::move( future ) );
}
@ -220,6 +237,14 @@ void IdentifierCompleter::ResultsForQueryAndType(
}
void IdentifierCompleter::ClearCandidatesStoredForFile(
const std::string &filetype,
const std::string &filepath )
{
GetCandidateList( filetype, filepath ).clear();
}
std::list< const Candidate* >& IdentifierCompleter::GetCandidateList(
const std::string &filetype,
const std::string &filepath )
@ -242,14 +267,19 @@ std::list< const Candidate* >& IdentifierCompleter::GetCandidateList(
void IdentifierCompleter::InitThreads()
{
int threads_to_create =
int query_threads_to_create =
std::max( MIN_ASYNC_THREADS,
std::min( MAX_ASYNC_THREADS, thread::hardware_concurrency() ) );
for ( int i = 0; i < threads_to_create; ++i )
for ( int i = 0; i < query_threads_to_create; ++i )
{
threads_.create_thread( bind( ThreadMain, boost::ref( latest_task_ ) ) );
query_threads_.create_thread( bind( QueryThreadMain,
boost::ref( latest_query_task_ ) ) );
}
buffer_identifiers_thread_ = boost::thread(
BufferIdentifiersThreadMain,
boost::ref( buffer_identifiers_task_stack_ ) );
}

33
cpp/ycm/IdentifierCompleter.h
View File

@ -19,6 +19,7 @@
#define COMPLETER_H_7AR4UGXE
#include "ConcurrentLatestValue.h"
#include "ConcurrentStack.h"
#include "Future.h"
#include <boost/utility.hpp>
@ -52,9 +53,15 @@ typedef boost::unordered_map< std::string,
typedef boost::unordered_map< std::string,
boost::shared_ptr< FilepathToCandidates > > FiletypeMap;
typedef ConcurrentLatestValue<
boost::shared_ptr<
boost::packaged_task< AsyncResults > > > LatestTask;
typedef boost::shared_ptr<
boost::packaged_task< AsyncResults > > QueryTask;
typedef ConcurrentLatestValue< QueryTask > LatestQueryTask;
typedef boost::shared_ptr< boost::packaged_task< void > > VoidTask;
typedef ConcurrentStack< VoidTask > BufferIdentifiersTaskStack;
class IdentifierCompleter : boost::noncopyable
{
@ -76,8 +83,13 @@ public:
const std::string &filetype,
const std::string &filepath );
void ClearCandidatesStoredForFile( const std::string &filetype,
const std::string &filepath );
// NOTE: params are taken by value on purpose! With a C++11 compiler we can
// avoid an expensive move of buffer_contents if the param is taken by value
// (move ctors FTW)
void AddCandidatesToDatabaseFromBufferAsync(
std::string buffer_contents,
std::string filetype,
std::string filepath );
// Only provided for tests!
std::vector< std::string > CandidatesForQuery(
@ -97,6 +109,9 @@ private:
const std::string &filetype,
std::vector< Result > &results ) const;
void ClearCandidatesStoredForFile( const std::string &filetype,
const std::string &filepath );
std::list< const Candidate* >& GetCandidateList(
const std::string &filetype,
const std::string &filepath );
@ -112,11 +127,15 @@ private:
FiletypeMap filetype_map_;
mutable LatestTask latest_task_;
mutable LatestQueryTask latest_query_task_;
BufferIdentifiersTaskStack buffer_identifiers_task_stack_;
bool threading_enabled_;
boost::thread_group threads_;
boost::thread_group query_threads_;
boost::thread buffer_identifiers_thread_;
};
} // namespace YouCompleteMe

7
cpp/ycm/IdentifierUtils.cpp
View File

@ -38,11 +38,10 @@ const char* COMMENT_AND_STRING_REGEX =
const char* IDENTIFIER_REGEX = "[_a-zA-Z]\\w*";
std::string RemoveIdentifierFreeText( const std::string &text )
std::string RemoveIdentifierFreeText( std::string text )
{
std::string new_text = text;
boost::erase_all_regex( new_text, boost::regex( COMMENT_AND_STRING_REGEX ) );
return new_text;
boost::erase_all_regex( text, boost::regex( COMMENT_AND_STRING_REGEX ) );
return text;
}

7
cpp/ycm/IdentifierUtils.h
View File

@ -24,7 +24,12 @@
namespace YouCompleteMe
{
std::string RemoveIdentifierFreeText( const std::string &text );
// NOTE: this function accepts the text param by value on purpose; it internally
// needs a copy before processing the text so the copy might as well be made on
// the parameter BUT if this code is compiled in C++11 mode a move constructor
// can be called on the passed-in value. This is not possible if we accept the
// param by const ref.
std::string RemoveIdentifierFreeText( std::string text );
std::vector< std::string > ExtractIdentifiersFromText(
const std::string &text );

4
cpp/ycm/indexer.cpp
View File

@ -59,8 +59,8 @@ BOOST_PYTHON_MODULE(indexer)
.def( "EnableThreading", &IdentifierCompleter::EnableThreading )
.def( "AddCandidatesToDatabase",
&IdentifierCompleter::AddCandidatesToDatabase )
.def( "AddCandidatesToDatabaseFromBuffer",
&IdentifierCompleter::AddCandidatesToDatabaseFromBuffer )
.def( "AddCandidatesToDatabaseFromBufferAsync",
&IdentifierCompleter::AddCandidatesToDatabaseFromBufferAsync )
.def( "CandidatesForQueryAndTypeAsync",
&IdentifierCompleter::CandidatesForQueryAndTypeAsync );

3
cpp/ycm/tests/IdentifierUtils_test.cpp
View File

@ -81,7 +81,8 @@ TEST( IdentifierUtilsTest, RemoveIdentifierFreeTextWorks )
TEST( IdentifierUtilsTest, ExtractIdentifiersFromTextWorks )
{
EXPECT_THAT( ExtractIdentifiersFromText(
"foo $_bar &BazGoo FOO= !!! '-' - _ (x) one-two !moo [qqq]" ),
"foo $_bar \n&BazGoo\n FOO= !!! '-' - _ (x) one-two !moo [qqq]"
),
ElementsAre( "foo",
"_bar",
"BazGoo",

2
python/ycm.py
View File

@ -85,7 +85,7 @@ class IdentifierCompleter( Completer ):
return
text = "\n".join( vim.current.buffer )
self.completer.AddCandidatesToDatabaseFromBuffer( text,
self.completer.AddCandidatesToDatabaseFromBufferAsync( text,
filetype,
filepath )