New system for specifying clang flags

Now the .ycm_clang_options file is a python script that needs to implement our
API. This enables the user to do arbitrary things when computing flags.

cpp/ycm/ClangCompleter.cpp

@@ -322,22 +322,6 @@ void ClangCompleter::EnableThreading()
 }
 
 
-void ClangCompleter::SetGlobalCompileFlags(
-    const std::vector< std::string > &flags )
-{
-  global_flags_ = flags;
-}
-
-
-void ClangCompleter::SetFileCompileFlags(
-    const std::string &filename,
-    const std::vector< std::string > &flags )
-{
-  flags_for_file_[ filename ] =
-    make_shared< std::vector< std::string > >( flags );
-}
-
-
 std::vector< Diagnostic > ClangCompleter::DiagnosticsForFile(
     const std::string &filename )
 {
@@ -377,7 +361,8 @@ bool ClangCompleter::UpdatingTranslationUnit()
 
 void ClangCompleter::UpdateTranslationUnit(
     const std::string &filename,
-    const std::vector< UnsavedFile > &unsaved_files )
+    const std::vector< UnsavedFile > &unsaved_files,
+    const std::vector< std::string > &flags )
 {
   TranslationUnitForFilename::iterator it =
     filename_to_translation_unit_.find( filename );
@@ -397,20 +382,22 @@ void ClangCompleter::UpdateTranslationUnit(
   else
   {
     filename_to_translation_unit_[ filename ] =
-      CreateTranslationUnit( filename, unsaved_files );
+      CreateTranslationUnit( filename, unsaved_files, flags );
   }
 }
 
 
 void ClangCompleter::UpdateTranslationUnitAsync(
     std::string filename,
-    std::vector< UnsavedFile > unsaved_files )
+    std::vector< UnsavedFile > unsaved_files,
+    std::vector< std::string > flags )
 {
   boost::function< void() > functor =
     bind( &ClangCompleter::UpdateTranslationUnit,
           boost::ref( *this ),
           boost::move( filename ),
-          boost::move( unsaved_files ) );
+          boost::move( unsaved_files ),
+          boost::move( flags ) );
 
   boost::lock_guard< boost::mutex > lock( file_parse_task_mutex_ );
 
@@ -428,7 +415,8 @@ std::vector< CompletionData > ClangCompleter::CandidatesForLocationInFile(
     const std::string &filename,
     int line,
     int column,
-    const std::vector< UnsavedFile > &unsaved_files )
+    const std::vector< UnsavedFile > &unsaved_files,
+    const std::vector< std::string > &flags )
 {
   std::vector< CXUnsavedFile > cxunsaved_files = ToCXUnsavedFiles(
       unsaved_files );
@@ -444,7 +432,9 @@ std::vector< CompletionData > ClangCompleter::CandidatesForLocationInFile(
   // call reparse*, but parse* which is even less efficient.
 
   CXCodeCompleteResults *results =
-    clang_codeCompleteAt( GetTranslationUnitForFile( filename, unsaved_files ),
+    clang_codeCompleteAt( GetTranslationUnitForFile( filename,
+                                                     unsaved_files,
+                                                     flags ),
                           filename.c_str(),
                           line,
                           column,
@@ -460,11 +450,12 @@ std::vector< CompletionData > ClangCompleter::CandidatesForLocationInFile(
 
 Future< AsyncCompletions >
 ClangCompleter::CandidatesForQueryAndLocationInFileAsync(
-    const std::string &query,
+    std::string query,
-    const std::string &filename,
+    std::string filename,
     int line,
     int column,
-    const std::vector< UnsavedFile > &unsaved_files )
+    std::vector< UnsavedFile > unsaved_files,
+    std::vector< std::string > flags )
 {
   // TODO: throw exception when threading is not enabled and this is called
   if ( !threading_enabled_ )
@@ -511,10 +502,11 @@ ClangCompleter::CandidatesForQueryAndLocationInFileAsync(
       candidates_for_location_in_file_functor =
       bind( &ClangCompleter::CandidatesForLocationInFile,
             boost::ref( *this ),
-            filename,
+            boost::move( filename ),
             line,
             column,
-            unsaved_files );
+            boost::move( unsaved_files ),
+            boost::move( flags ) );
 
     shared_ptr< packaged_task< AsyncCompletions > > task =
       make_shared< packaged_task< AsyncCompletions > >(
@@ -530,14 +522,15 @@ ClangCompleter::CandidatesForQueryAndLocationInFileAsync(
 
 CXTranslationUnit ClangCompleter::CreateTranslationUnit(
     const std::string &filename,
-    const std::vector< UnsavedFile > &unsaved_files )
+    const std::vector< UnsavedFile > &unsaved_files,
+    const std::vector< std::string > &flags )
 {
-  std::vector< const char* > flags = FlagsForFilename( filename );
+  std::vector< const char* > pointer_flags;
-  flags.reserve( flags.size() + global_flags_.size() );
+  pointer_flags.reserve( flags.size() );
 
-  foreach ( const std::string &flag, global_flags_ )
+  foreach ( const std::string &flag, flags )
   {
-    flags.push_back( flag.c_str() );
+    pointer_flags.push_back( flag.c_str() );
   }
 
   std::vector< CXUnsavedFile > cxunsaved_files = ToCXUnsavedFiles(
@@ -546,8 +539,8 @@ CXTranslationUnit ClangCompleter::CreateTranslationUnit(
   CXTranslationUnit unit = clang_parseTranslationUnit(
       clang_index_,
       filename.c_str(),
-      &flags[ 0 ],
+      &pointer_flags[ 0 ],
-      flags.size(),
+      pointer_flags.size(),
       &cxunsaved_files[ 0 ],
       cxunsaved_files.size(),
       clang_defaultEditingTranslationUnitOptions() );
@@ -564,41 +557,10 @@ CXTranslationUnit ClangCompleter::CreateTranslationUnit(
 }
 
 
-// The implementation of this function is somewhat non-obvious because we need
-// to make sure that the data pointed to by the const char* pointers returned
-// outlives this function. We want to make sure that we are calling c_str on the
-// string objects that are actually stored in flags_for_file_
-std::vector< const char* > ClangCompleter::FlagsForFilename(
-    const std::string &filename)
-{
-  FlagsForFile::iterator it =
-    flags_for_file_.find( filename );
-
-  if ( it == flags_for_file_.end() )
-  {
-    flags_for_file_[ filename ] = make_shared< std::vector< std::string > >(
-        SanitizeClangFlags(
-            SplitFlags(
-                GetNearestClangOptions( filename ) ) ) );
-
-    it = flags_for_file_.find( filename );
-  }
-
-  // TODO: assert it != end
-
-  std::vector< const char* > flags;
-  foreach ( const std::string &flag, *it->second )
-  {
-    flags.push_back( flag.c_str() );
-  }
-
-  return flags;
-}
-
-
 CXTranslationUnit ClangCompleter::GetTranslationUnitForFile(
     const std::string &filename,
-    const std::vector< UnsavedFile > &unsaved_files )
+    const std::vector< UnsavedFile > &unsaved_files,
+    const std::vector< std::string > &flags )
 {
   TranslationUnitForFilename::iterator it =
     filename_to_translation_unit_.find( filename );
@@ -606,7 +568,9 @@ CXTranslationUnit ClangCompleter::GetTranslationUnitForFile(
   if ( it != filename_to_translation_unit_.end() )
     return it->second;
 
-  CXTranslationUnit unit = CreateTranslationUnit( filename, unsaved_files );
+  CXTranslationUnit unit = CreateTranslationUnit( filename,
+                                                  unsaved_files,
+                                                  flags );
   filename_to_translation_unit_[ filename ] = unit;
   return unit;
 }

cpp/ycm/ClangCompleter.h

@@ -56,34 +56,37 @@ public:
 
   void EnableThreading();
 
-  void SetGlobalCompileFlags( const std::vector< std::string > &flags );
-
-  void SetFileCompileFlags( const std::string &filename,
-                            const std::vector< std::string > &flags );
-
   std::vector< Diagnostic > DiagnosticsForFile( const std::string &filename );
 
   bool UpdatingTranslationUnit();
 
   void UpdateTranslationUnit( const std::string &filename,
-                              const std::vector< UnsavedFile > &unsaved_files );
+                              const std::vector< UnsavedFile > &unsaved_files,
+                              const std::vector< std::string > &flags );
 
+  // NOTE: params are taken by value on purpose! With a C++11 compiler we can
+  // avoid internal copies if params are taken by value (move ctors FTW)
   void UpdateTranslationUnitAsync(
       std::string filename,
-      std::vector< UnsavedFile > unsaved_files );
+      std::vector< UnsavedFile > unsaved_files,
+      std::vector< std::string > flags );
 
   std::vector< CompletionData > CandidatesForLocationInFile(
       const std::string &filename,
       int line,
       int column,
-      const std::vector< UnsavedFile > &unsaved_files );
+      const std::vector< UnsavedFile > &unsaved_files,
+      const std::vector< std::string > &flags );
 
+  // NOTE: params are taken by value on purpose! With a C++11 compiler we can
+  // avoid internal copies if params are taken by value (move ctors FTW)
   Future< AsyncCompletions > CandidatesForQueryAndLocationInFileAsync(
-      const std::string &query,
+      std::string query,
-      const std::string &filename,
+      std::string filename,
       int line,
       int column,
-      const std::vector< UnsavedFile > &unsaved_files );
+      std::vector< UnsavedFile > unsaved_files,
+      std::vector< std::string > flags );
 
 private:
   typedef ConcurrentLatestValue<
@@ -93,13 +96,13 @@ private:
   // caller takes ownership of translation unit
   CXTranslationUnit CreateTranslationUnit(
       const std::string &filename,
-      const std::vector< UnsavedFile > &unsaved_files );
+      const std::vector< UnsavedFile > &unsaved_files,
-
+      const std::vector< std::string > &flags );
-  std::vector< const char* > FlagsForFilename( const std::string &filename );
 
   CXTranslationUnit GetTranslationUnitForFile(
       const std::string &filename,
-      const std::vector< UnsavedFile > &unsaved_files );
+      const std::vector< UnsavedFile > &unsaved_files,
+      const std::vector< std::string > &flags );
 
   std::vector< CompletionData > SortCandidatesForQuery(
       const std::string &query,
@@ -120,12 +123,8 @@ private:
 
   CXIndex clang_index_;
 
-  FlagsForFile flags_for_file_;
-
   TranslationUnitForFilename filename_to_translation_unit_;
 
-  std::vector< std::string > global_flags_;
-
   CandidateRepository &candidate_repository_;
 
   // TODO: move everything thread-related into a separated helper class

cpp/ycm/ClangUtils.h

@@ -25,6 +25,7 @@ namespace YouCompleteMe
 {
 
 // TODO: move most of the anon funcs in ClangCompleter.cpp to here
+// TODO: remove these functions
 
 // Removes flags that are either unnecessary or cause clang to crash
 std::vector< std::string > SanitizeClangFlags(

cpp/ycm/IdentifierCompleter.h

@@ -82,7 +82,7 @@ public:
                                           const std::string &filepath );
 
   // NOTE: params are taken by value on purpose! With a C++11 compiler we can
-  // avoid an expensive move of buffer_contents if the param is taken by value
+  // avoid an expensive copy of buffer_contents if the param is taken by value
   // (move ctors FTW)
   void AddCandidatesToDatabaseFromBufferAsync(
       std::string buffer_contents,

cpp/ycm/indexer.cpp

@@ -95,8 +95,6 @@ BOOST_PYTHON_MODULE(indexer)
 
   class_< ClangCompleter, boost::noncopyable >( "ClangCompleter" )
     .def( "EnableThreading", &ClangCompleter::EnableThreading )
-    .def( "SetGlobalCompileFlags", &ClangCompleter::SetGlobalCompileFlags )
-    .def( "SetFileCompileFlags", &ClangCompleter::SetFileCompileFlags )
     .def( "DiagnosticsForFile", &ClangCompleter::DiagnosticsForFile )
     .def( "UpdatingTranslationUnit", &ClangCompleter::UpdatingTranslationUnit )
     .def( "UpdateTranslationUnit", &ClangCompleter::UpdateTranslationUnit )

python/ycm.py

@@ -20,11 +20,16 @@
 import vim
 import indexer
 import abc
+import imp
+import string
+import random
+import os
 
 MIN_NUM_CHARS = int( vim.eval( "g:ycm_min_num_of_chars_for_completion" ) )
 CLANG_COMPLETION_ENABLED = int( vim.eval( "g:ycm_clang_completion_enabled" ) )
 CLANG_FILETYPES = set( [ 'c', 'cpp', 'objc', 'objcpp' ] )
 MAX_IDENTIFIER_COMPLETIONS_RETURNED = 10
+CLANG_OPTIONS_FILENAME = '.ycm_clang_options'
 
 
 class Completer( object ):
@@ -134,6 +139,7 @@ class ClangCompleter( Completer ):
     self.filename_holder = []
     self.last_diagnostics = []
     self.possibly_new_diagnostics = False
+    self.flags = Flags()
 
 
   def GetUnsavedFilesVector( self ):
@@ -188,7 +194,8 @@ class ClangCompleter( Completer ):
       current_buffer.name,
       line,
       column,
-      files )
+      files,
+      self.flags.FlagsForFile( current_buffer.name ) )
 
 
   def CandidatesFromStoredRequest( self ):
@@ -203,9 +210,14 @@ class ClangCompleter( Completer ):
   def OnFileReadyToParse( self ):
     if NumLinesInBuffer( vim.current.buffer ) < 5:
       return
+
     self.possibly_new_diagnostics = True
-    self.completer.UpdateTranslationUnitAsync( vim.current.buffer.name,
+
-                                               self.GetUnsavedFilesVector() )
+    filename = vim.current.buffer.name
+    self.completer.UpdateTranslationUnitAsync(
+      filename,
+      self.GetUnsavedFilesVector(),
+      self.flags.FlagsForFile( filename ) )
 
 
   def DiagnosticsForCurrentFileReady( self ):
@@ -222,6 +234,92 @@ class ClangCompleter( Completer ):
     return self.last_diagnostics
 
 
+class Flags( object ):
+  def __init__( self ):
+    # It's caches all the way down...
+    self.flags_for_file = {}
+    self.flags_module_for_file = {}
+    self.flags_module_for_flags_module_file = {}
+
+
+  def FlagsForFile( self, filename ):
+    try:
+      return self.flags_for_file[ filename ]
+    except KeyError:
+      flags_module = self.FlagsModuleForFile( filename )
+      if not flags_module:
+        return indexer.StringVec()
+
+      results = flags_module.FlagsForFile( filename )
+      sanitized_flags = SanitizeFlags( results[ 'flags' ] )
+
+      if results[ 'do_cache' ]:
+        self.flags_for_file[ filename ] = sanitized_flags
+      return sanitized_flags
+
+
+  def FlagsModuleForFile( self, filename ):
+    try:
+      return self.flags_module_for_file[ filename ]
+    except KeyError:
+      flags_module_file = FlagsModuleSourceFileForFile( filename )
+      if not flags_module_file:
+        return None
+
+      try:
+        flags_module = self.flags_module_for_flags_module_file[
+          flags_module_file ]
+      except KeyError:
+        flags_module = imp.load_source( RandomName(), flags_module_file )
+        self.flags_module_for_flags_module_file[
+          flags_module_file ] = flags_module
+
+      self.flags_module_for_file[ filename ] = flags_module
+      return flags_module
+
+
+def FlagsModuleSourceFileForFile( filename ):
+  parent_folder = os.path.dirname( filename )
+  old_parent_folder = ''
+
+  while True:
+    current_file = os.path.join( parent_folder, CLANG_OPTIONS_FILENAME )
+    if os.path.exists( current_file ):
+      return current_file
+
+    old_parent_folder = parent_folder
+    parent_folder = os.path.dirname( parent_folder )
+
+    if parent_folder == old_parent_folder:
+      return None
+
+
+
+def RandomName():
+  return ''.join( random.choice( string.ascii_lowercase ) for x in range( 15 ) )
+
+
+def SanitizeFlags( flags ):
+  sanitized_flags = []
+  saw_arch = False
+  for i, flag in enumerate( flags ):
+    if flag == '-arch':
+      saw_arch = True
+      continue
+    elif flag.startswith( '-arch' ):
+      continue
+    elif saw_arch:
+      saw_arch = False
+      continue
+
+    sanitized_flags.append( flag )
+
+  vector = indexer.StringVec()
+  for flag in sanitized_flags:
+    vector.append( flag )
+  return vector
+
+
 def NumLinesInBuffer( buffer ):
   # This is actually less than obvious, that's why it's wrapped in a function
   return len( buffer )