Auto merge of #2563 - micbou:import-requests-lazily, r=puremourning

[READY] Import the requests module lazily

The requests module is slow to import. See https://github.com/kennethreitz/requests/issues/3213. We should lazy load it to improve startup time. We do that by adding two methods to the `BaseRequest` class: one that returns the requests module and another the session object since it depends on the `request-futures` module and `requests-futures` imports `requests`. In addition, we make sure that no requests are sent at startup otherwise there would be no point to lazy load these. These requests would fail anyway since the server can't be ready yet.

Here are the improvements on startup time:
<table>
  <tr>
    <th rowspan="2">Platform</th>
    <th colspan="2">First run (ms)</th>
    <th colspan="2">Subsequent runs (ms)</th>
  </tr>
  <tr>
    <td>Before</td>
    <td>After</td>
    <td>Before</td>
    <td>After</td>
  </tr>
  <tr>
    <td>Ubuntu 16.04 64-bit</td>
    <td>240</td>
    <td>131</td>
    <td>173</td>
    <td>74</td>
  </tr>
  <tr>
    <td>macOS 10.12</td>
    <td>435</td>
    <td>315</td>
    <td>261</td>
    <td>208</td>
  </tr>
  <tr>
    <td>Windows 10 64-bit</td>
    <td>894</td>
    <td>594</td>
    <td>359</td>
    <td>247</td>
  </tr>
</table>

*Results obtained by running the `prof.py` script from [this branch](https://github.com/micbou/YouCompleteMe/tree/profiling-startup). The difference between first run and subsequent runs is Python bytecode generation (`*.pyc` files).*

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/valloric/youcompleteme/2563)
<!-- Reviewable:end -->

autoload/youcompleteme.vim

@@ -82,7 +82,7 @@ function! youcompleteme#Enable()
     " Note that these events will NOT trigger for the file vim is started with;
     " so if you do "vim foo.cc", these events will not trigger when that buffer
     " is read. This is because youcompleteme#Enable() is called on VimEnter and
-    " that happens *after" BufRead/BufEnter has already triggered for the
+    " that happens *after* BufRead/FileType has already triggered for the
     " initial file.
     " We also need to trigger buf init code on the FileType event because when
     " the user does :enew and then :set ft=something, we need to run buf init
@@ -97,10 +97,15 @@ function! youcompleteme#Enable()
     autocmd CompleteDone * call s:OnCompleteDone()
   augroup END
 
-  " Calling this once solves the problem of BufRead/BufEnter not triggering for
+  " BufRead/FileType events are not triggered for the first loaded file.
-  " the first loaded file. This should be the last command executed in this
+  " However, we don't directly call the s:OnBufferRead function because it would
-  " function!
+  " send requests that can't succeed as the server is not ready yet and would
-  call s:OnBufferRead()
+  " slow down startup.
+  call s:DisableOnLargeFile( expand( '%' ) )
+
+  if s:AllowedToCompleteInCurrentBuffer()
+    call s:SetCompleteFunc()
+  endif
 endfunction
 
 

python/ycm/client/base_request.py

@@ -25,20 +25,16 @@ from builtins import *  # noqa
 
 import contextlib
 import logging
-import requests
 import urllib.parse
 import json
 from future.utils import native
 from base64 import b64decode, b64encode
-from requests_futures.sessions import FuturesSession
-from ycm.unsafe_thread_pool_executor import UnsafeThreadPoolExecutor
 from ycm import vimsupport
 from ycmd.utils import ToBytes
 from ycmd.hmac_utils import CreateRequestHmac, CreateHmac, SecureBytesEqual
 from ycmd.responses import ServerError, UnknownExtraConf
 
 _HEADERS = {'content-type': 'application/json'}
-_EXECUTOR = UnsafeThreadPoolExecutor( max_workers = 30 )
 _CONNECT_TIMEOUT_SEC = 0.01
 # Setting this to None seems to screw up the Requests/urllib3 libs.
 _READ_TIMEOUT_SEC = 30
@@ -104,14 +100,14 @@ class BaseRequest( object ):
     request_uri = _BuildUri( handler )
     if method == 'POST':
       sent_data = _ToUtf8Json( data )
-      return BaseRequest.session.post(
+      return BaseRequest.Session().post(
           request_uri,
           data = sent_data,
           headers = BaseRequest._ExtraHeaders( method,
                                                request_uri,
                                                sent_data ),
           timeout = ( _CONNECT_TIMEOUT_SEC, timeout ) )
-    return BaseRequest.session.get(
+    return BaseRequest.Session().get(
         request_uri,
         headers = BaseRequest._ExtraHeaders( method, request_uri ),
         timeout = ( _CONNECT_TIMEOUT_SEC, timeout ) )
@@ -129,7 +125,31 @@ class BaseRequest( object ):
                            BaseRequest.hmac_secret ) )
     return headers
 
-  session = FuturesSession( executor = _EXECUTOR )
+
+  # These two methods exist to avoid importing the requests module at startup;
+  # reducing loading time since this module is slow to import.
+  @classmethod
+  def Requests( cls ):
+    try:
+      return cls.requests
+    except AttributeError:
+      import requests
+      cls.requests = requests
+      return requests
+
+
+  @classmethod
+  def Session( cls ):
+    try:
+      return cls.session
+    except AttributeError:
+      from ycm.unsafe_thread_pool_executor import UnsafeThreadPoolExecutor
+      from requests_futures.sessions import FuturesSession
+      executor = UnsafeThreadPoolExecutor( max_workers = 30 )
+      cls.session = FuturesSession( executor = executor )
+      return cls.session
+
+
   server_location = ''
   hmac_secret = ''
 
@@ -161,7 +181,7 @@ def BuildRequestData( filepath = None ):
 def JsonFromFuture( future ):
   response = future.result()
   _ValidateResponseObject( response )
-  if response.status_code == requests.codes.server_error:
+  if response.status_code == BaseRequest.Requests().codes.server_error:
     raise MakeServerException( response.json() )
 
   # We let Requests handle the other status types, we only handle the 500
@@ -199,7 +219,7 @@ def HandleServerException( display = True, truncate = False ):
         _LoadExtraConfFile( e.extra_conf_file )
       else:
         _IgnoreExtraConfFile( e.extra_conf_file )
-  except requests.exceptions.ConnectionError:
+  except BaseRequest.Requests().exceptions.ConnectionError:
     # We don't display this exception to the user since it is likely to happen
     # for each subsequent request (typically if the server crashed) and we
     # don't want to spam the user with it.