[enh] use one single http connection pool : improve response time. close #100

searx/autocomplete.py

 
 
 from lxml import etree
-from requests import get
 from json import loads
 from urllib import urlencode
 from searx.languages import language_codes
 from searx.engines import (
     categories, engines, engine_shortcuts
 )
+from searx.poolrequests import get
 
 
 def searx_bang(full_query):

searx/engines/wikidata.py

 import json
-from requests import get
 from urllib import urlencode
+from searx.poolrequests import get
 from searx.utils import format_date_by_locale
 
 result_count = 1

searx/poolrequests.py

+import requests
+
+
+the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
+the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
+
+
+class SessionSinglePool(requests.Session):
+
+    def __init__(self):
+        global the_https_adapter, the_http_adapter
+        super(SessionSinglePool, self).__init__()
+
+        # reuse the same adapters
+        self.adapters.clear()
+        self.mount('https://', the_https_adapter)
+        self.mount('http://', the_http_adapter)
+
+    def close(self):
+        """Call super, but clear adapters since there are managed globaly"""
+        self.adapters.clear()
+        super(SessionSinglePool, self).close()
+
+
+def request(method, url, **kwargs):
+    """same as requests/requests/api.py request(...) except it use SessionSinglePool"""
+    session = SessionSinglePool()
+    response = session.request(method=method, url=url, **kwargs)
+    session.close()
+    return response
+
+
+def get(url, **kwargs):
+    kwargs.setdefault('allow_redirects', True)
+    return request('get', url, **kwargs)
+
+
+def options(url, **kwargs):
+    kwargs.setdefault('allow_redirects', True)
+    return request('options', url, **kwargs)
+
+
+def head(url, **kwargs):
+    kwargs.setdefault('allow_redirects', False)
+    return request('head', url, **kwargs)
+
+
+def post(url, data=None, json=None, **kwargs):
+    return request('post', url, data=data, json=json, **kwargs)
+
+
+def put(url, data=None, **kwargs):
+    return request('put', url, data=data, **kwargs)
+
+
+def patch(url, data=None, **kwargs):
+    return request('patch', url, data=data, **kwargs)
+
+
+def delete(url, **kwargs):
+    return request('delete', url, **kwargs)

searx/search.py

 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
 '''
 
-import requests as requests_lib
 import threading
 import re
+import searx.poolrequests as requests_lib
 from itertools import izip_longest, chain
 from operator import itemgetter
 from Queue import Queue
 from searx.query import Query
 from searx import logger
 
-
 logger = logger.getChild('search')
 
 number_of_searches = 0

searx/webapp.py

 import os
 
 from datetime import datetime, timedelta
-from requests import get as http_get
 from itertools import chain
 from urllib import urlencode
 from flask import (
 )
 from flask.ext.babel import Babel, gettext, format_date
 from searx import settings, searx_dir
+from searx.poolrequests import get as http_get
 from searx.engines import (
     categories, engines, get_engines_stats, engine_shortcuts
 )