Buffer identifiers are now extracted async

cpp/ycm/ConcurrentStack.h

@@ -0,0 +1,66 @@
+// Copyright (C) 2011, 2012  Strahinja Val Markovic  <val@markovic.io>
+//
+// This file is part of YouCompleteMe.
+//
+// YouCompleteMe is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// YouCompleteMe is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with YouCompleteMe.  If not, see <http://www.gnu.org/licenses/>.
+
+#ifndef CONCURRENTSTACK_H_TGI0GOR6
+#define CONCURRENTSTACK_H_TGI0GOR6
+
+#include <boost/thread.hpp>
+#include <boost/utility.hpp>
+#include <stack>
+
+namespace YouCompleteMe
+{
+
+template <typename T>
+class ConcurrentStack : boost::noncopyable
+{
+public:
+
+  void Push( const T& data )
+  {
+    {
+      boost::unique_lock< boost::mutex > lock( mutex_ );
+      stack_.push( data );
+    }
+
+    condition_variable_.notify_one();
+  }
+
+  T Pop()
+  {
+    boost::unique_lock< boost::mutex > lock( mutex_ );
+
+    while ( stack_.empty() )
+    {
+      condition_variable_.wait( lock );
+    }
+
+    T top = stack_.top();
+    stack_.pop();
+    return top;
+  }
+
+private:
+  std::stack< T > stack_;
+  boost::mutex mutex_;
+  boost::condition_variable condition_variable_;
+
+};
+
+} // namespace YouCompleteMe
+
+#endif /* end of include guard: CONCURRENTSTACK_H_TGI0GOR6 */

cpp/ycm/IdentifierCompleter.cpp

@@ -44,11 +44,21 @@ extern const unsigned int MIN_ASYNC_THREADS = 2;
 namespace
 {
 
-void ThreadMain( LatestTask &latest_task )
+void QueryThreadMain( LatestQueryTask &latest_query_task )
 {
   while ( true )
   {
-    ( *latest_task.Get() )();
+    ( *latest_query_task.Get() )();
+  }
+
+}
+
+void BufferIdentifiersThreadMain(
+  BufferIdentifiersTaskStack &buffer_identifiers_task_stack )
+{
+  while ( true )
+  {
+    ( *buffer_identifiers_task_stack.Pop() )();
   }
 }
 
@@ -123,11 +133,20 @@ void IdentifierCompleter::AddCandidatesToDatabaseFromBuffer(
 }
 
 
-void IdentifierCompleter::ClearCandidatesStoredForFile(
+void IdentifierCompleter::AddCandidatesToDatabaseFromBufferAsync(
-    const std::string &filetype,
+    std::string buffer_contents,
-    const std::string &filepath )
+    std::string filetype,
+    std::string filepath )
 {
-  GetCandidateList( filetype, filepath ).clear();
+  boost::function< void() > functor =
+    bind( &IdentifierCompleter::AddCandidatesToDatabaseFromBuffer,
+          boost::ref( *this ),
+          boost::move( buffer_contents ),
+          boost::move( filetype ),
+          boost::move( filepath ) );
+
+  buffer_identifiers_task_stack_.Push(
+      make_shared< packaged_task< void > >( functor ) );
 }
 
 
@@ -170,15 +189,13 @@ Future< AsyncResults > IdentifierCompleter::CandidatesForQueryAndTypeAsync(
           query,
           filetype );
 
-  // Try not to look at this too hard, it may burn your eyes.
+  QueryTask task = make_shared< packaged_task< AsyncResults > >(
-  shared_ptr< packaged_task< AsyncResults > > task =
-    make_shared< packaged_task< AsyncResults > >(
       bind( ReturnValueAsShared< std::vector< std::string > >,
             functor ) );
 
   unique_future< AsyncResults > future = task->get_future();
 
-  latest_task_.Set( task );
+  latest_query_task_.Set( task );
   return Future< AsyncResults >( boost::move( future ) );
 }
 
@@ -220,6 +237,14 @@ void IdentifierCompleter::ResultsForQueryAndType(
 }
 
 
+void IdentifierCompleter::ClearCandidatesStoredForFile(
+    const std::string &filetype,
+    const std::string &filepath )
+{
+  GetCandidateList( filetype, filepath ).clear();
+}
+
+
 std::list< const Candidate* >& IdentifierCompleter::GetCandidateList(
     const std::string &filetype,
     const std::string &filepath )
@@ -242,14 +267,19 @@ std::list< const Candidate* >& IdentifierCompleter::GetCandidateList(
 
 void IdentifierCompleter::InitThreads()
 {
-  int threads_to_create =
+  int query_threads_to_create =
     std::max( MIN_ASYNC_THREADS,
       std::min( MAX_ASYNC_THREADS, thread::hardware_concurrency() ) );
 
-  for ( int i = 0; i < threads_to_create; ++i )
+  for ( int i = 0; i < query_threads_to_create; ++i )
   {
-    threads_.create_thread( bind( ThreadMain, boost::ref( latest_task_ ) ) );
+    query_threads_.create_thread( bind( QueryThreadMain,
+                                        boost::ref( latest_query_task_ ) ) );
   }
+
+  buffer_identifiers_thread_ = boost::thread(
+      BufferIdentifiersThreadMain,
+      boost::ref( buffer_identifiers_task_stack_ ) );
 }
 
 

cpp/ycm/IdentifierCompleter.h

@@ -19,6 +19,7 @@
 #define COMPLETER_H_7AR4UGXE
 
 #include "ConcurrentLatestValue.h"
+#include "ConcurrentStack.h"
 #include "Future.h"
 
 #include <boost/utility.hpp>
@@ -52,9 +53,15 @@ typedef boost::unordered_map< std::string,
 typedef boost::unordered_map< std::string,
           boost::shared_ptr< FilepathToCandidates > > FiletypeMap;
 
-typedef ConcurrentLatestValue<
+typedef boost::shared_ptr<
-          boost::shared_ptr<
+            boost::packaged_task< AsyncResults > > QueryTask;
-            boost::packaged_task< AsyncResults > > > LatestTask;
+
+typedef ConcurrentLatestValue< QueryTask > LatestQueryTask;
+
+typedef boost::shared_ptr< boost::packaged_task< void > > VoidTask;
+
+typedef ConcurrentStack< VoidTask > BufferIdentifiersTaskStack;
+
 
 class IdentifierCompleter : boost::noncopyable
 {
@@ -76,8 +83,13 @@ public:
                                           const std::string &filetype,
                                           const std::string &filepath );
 
-  void ClearCandidatesStoredForFile( const std::string &filetype,
+  // NOTE: params are taken by value on purpose! With a C++11 compiler we can
-                                     const std::string &filepath );
+  // avoid an expensive move of buffer_contents if the param is taken by value
+  // (move ctors FTW)
+  void AddCandidatesToDatabaseFromBufferAsync(
+      std::string buffer_contents,
+      std::string filetype,
+      std::string filepath );
 
   // Only provided for tests!
 	std::vector< std::string > CandidatesForQuery(
@@ -97,6 +109,9 @@ private:
                                const std::string &filetype,
                                std::vector< Result > &results ) const;
 
+  void ClearCandidatesStoredForFile( const std::string &filetype,
+                                     const std::string &filepath );
+
   std::list< const Candidate* >& GetCandidateList(
       const std::string &filetype,
       const std::string &filepath );
@@ -112,11 +127,15 @@ private:
 
   FiletypeMap filetype_map_;
 
-  mutable LatestTask latest_task_;
+  mutable LatestQueryTask latest_query_task_;
+
+  BufferIdentifiersTaskStack buffer_identifiers_task_stack_;
 
   bool threading_enabled_;
 
-  boost::thread_group threads_;
+  boost::thread_group query_threads_;
+
+  boost::thread buffer_identifiers_thread_;
 };
 
 } // namespace YouCompleteMe

cpp/ycm/IdentifierUtils.cpp

@@ -38,11 +38,10 @@ const char* COMMENT_AND_STRING_REGEX =
 const char* IDENTIFIER_REGEX = "[_a-zA-Z]\\w*";
 
 
-std::string RemoveIdentifierFreeText( const std::string &text )
+std::string RemoveIdentifierFreeText( std::string text )
 {
-  std::string new_text = text;
+  boost::erase_all_regex( text, boost::regex( COMMENT_AND_STRING_REGEX ) );
-  boost::erase_all_regex( new_text, boost::regex( COMMENT_AND_STRING_REGEX ) );
+  return text;
-  return new_text;
 }
 
 

cpp/ycm/IdentifierUtils.h

@@ -24,7 +24,12 @@
 namespace YouCompleteMe
 {
 
-std::string RemoveIdentifierFreeText( const std::string &text );
+// NOTE: this function accepts the text param by value on purpose; it internally
+// needs a copy before processing the text so the copy might as well be made on
+// the parameter BUT if this code is compiled in C++11 mode a move constructor
+// can be called on the passed-in value. This is not possible if we accept the
+// param by const ref.
+std::string RemoveIdentifierFreeText( std::string text );
 
 std::vector< std::string > ExtractIdentifiersFromText(
     const std::string &text );

cpp/ycm/indexer.cpp

@@ -59,8 +59,8 @@ BOOST_PYTHON_MODULE(indexer)
     .def( "EnableThreading", &IdentifierCompleter::EnableThreading )
     .def( "AddCandidatesToDatabase",
           &IdentifierCompleter::AddCandidatesToDatabase )
-    .def( "AddCandidatesToDatabaseFromBuffer",
+    .def( "AddCandidatesToDatabaseFromBufferAsync",
-          &IdentifierCompleter::AddCandidatesToDatabaseFromBuffer )
+          &IdentifierCompleter::AddCandidatesToDatabaseFromBufferAsync )
     .def( "CandidatesForQueryAndTypeAsync",
           &IdentifierCompleter::CandidatesForQueryAndTypeAsync );
 

cpp/ycm/tests/IdentifierUtils_test.cpp

@@ -81,7 +81,8 @@ TEST( IdentifierUtilsTest, RemoveIdentifierFreeTextWorks )
 TEST( IdentifierUtilsTest, ExtractIdentifiersFromTextWorks )
 {
 	EXPECT_THAT( ExtractIdentifiersFromText(
-	             "foo $_bar &BazGoo FOO= !!! '-' - _ (x) one-two !moo [qqq]" ),
+	             "foo $_bar \n&BazGoo\n FOO= !!! '-' - _ (x) one-two !moo [qqq]"
+	             ),
 	             ElementsAre( "foo",
 	                          "_bar",
 	                          "BazGoo",

python/ycm.py

@@ -85,9 +85,9 @@ class IdentifierCompleter( Completer ):
       return
 
     text = "\n".join( vim.current.buffer )
-    self.completer.AddCandidatesToDatabaseFromBuffer( text,
+    self.completer.AddCandidatesToDatabaseFromBufferAsync( text,
-                                                      filetype,
+                                                           filetype,
-                                                      filepath )
+                                                           filepath )
 
 
   def OnFileEnter( self ):