Bladeren bron

[enh] gevent/grequests changed to the built-in threading lib

Adam Tauber 10 jaren geleden
bovenliggende
commit
d959cb1c05
5 gewijzigde bestanden met toevoegingen van 20 en 23 verwijderingen
  1. 1
    1
      requirements.txt
  2. 18
    5
      searx/search.py
  3. 0
    4
      searx/webapp.py
  4. 1
    1
      setup.py
  5. 0
    12
      versions.cfg

+ 1
- 1
requirements.txt Bestand weergeven

1
 flask
1
 flask
2
 flask-babel
2
 flask-babel
3
-grequests
3
+requests
4
 lxml
4
 lxml
5
 pyyaml
5
 pyyaml
6
 python-dateutil
6
 python-dateutil

+ 18
- 5
searx/search.py Bestand weergeven

15
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
15
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
16
 '''
16
 '''
17
 
17
 
18
-import grequests
18
+import requests as requests_lib
19
+import threading
19
 import re
20
 import re
20
 from itertools import izip_longest, chain
21
 from itertools import izip_longest, chain
21
 from datetime import datetime
22
 from datetime import datetime
32
 number_of_searches = 0
33
 number_of_searches = 0
33
 
34
 
34
 
35
 
36
+def threaded_requests(requests):
37
+    for fn, url, request_args in requests:
38
+        th = threading.Thread(
39
+            target=fn, args=(url,), kwargs=request_args, name=url,
40
+        )
41
+        th.start()
42
+
43
+    for th in threading.enumerate():
44
+        if th.name.startswith('http'):
45
+            th.join()
46
+
47
+
35
 # get default reqest parameter
48
 # get default reqest parameter
36
 def default_request_params():
49
 def default_request_params():
37
     return {
50
     return {
471
 
484
 
472
             # specific type of request (GET or POST)
485
             # specific type of request (GET or POST)
473
             if request_params['method'] == 'GET':
486
             if request_params['method'] == 'GET':
474
-                req = grequests.get
487
+                req = requests_lib.get
475
             else:
488
             else:
476
-                req = grequests.post
489
+                req = requests_lib.post
477
                 request_args['data'] = request_params['data']
490
                 request_args['data'] = request_params['data']
478
 
491
 
479
             # ignoring empty urls
492
             # ignoring empty urls
481
                 continue
494
                 continue
482
 
495
 
483
             # append request to list
496
             # append request to list
484
-            requests.append(req(request_params['url'], **request_args))
497
+            requests.append((req, request_params['url'], request_args))
485
 
498
 
486
         # send all search-request
499
         # send all search-request
487
-        grequests.map(requests)
500
+        threaded_requests(requests)
488
 
501
 
489
         # update engine-specific stats
502
         # update engine-specific stats
490
         for engine_name, engine_results in results.items():
503
         for engine_name, engine_results in results.items():

+ 0
- 4
searx/webapp.py Bestand weergeven

17
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
17
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
18
 '''
18
 '''
19
 
19
 
20
-from gevent import monkey
21
-monkey.patch_all()
22
-
23
-
24
 if __name__ == '__main__':
20
 if __name__ == '__main__':
25
     from sys import path
21
     from sys import path
26
     from os.path import realpath, dirname
22
     from os.path import realpath, dirname

+ 1
- 1
setup.py Bestand weergeven

41
     install_requires=[
41
     install_requires=[
42
         'flask',
42
         'flask',
43
         'flask-babel',
43
         'flask-babel',
44
-        'grequests',
44
+        'requests',
45
         'lxml',
45
         'lxml',
46
         'pyyaml',
46
         'pyyaml',
47
         'setuptools',
47
         'setuptools',

+ 0
- 12
versions.cfg Bestand weergeven

37
 beautifulsoup4 = 4.3.2
37
 beautifulsoup4 = 4.3.2
38
 
38
 
39
 # Required by:
39
 # Required by:
40
-# grequests==0.2.0
41
-gevent = 1.0
42
-
43
-# Required by:
44
-# gevent==1.0
45
-greenlet = 0.4.2
46
-
47
-# Required by:
48
-# searx==0.1
49
-grequests = 0.2.0
50
-
51
-# Required by:
52
 # robotframework-httplibrary==0.4.2
40
 # robotframework-httplibrary==0.4.2
53
 jsonpatch = 1.3
41
 jsonpatch = 1.3
54
 
42