瀏覽代碼

[fix] proper search timeout handling

Adam Tauber 10 年之前
父節點
當前提交
a5324d9d82
共有 1 個文件被更改,包括 19 次插入6 次删除
  1. 19
    6
      searx/search.py

+ 19
- 6
searx/search.py 查看文件

21
 from itertools import izip_longest, chain
21
 from itertools import izip_longest, chain
22
 from datetime import datetime
22
 from datetime import datetime
23
 from operator import itemgetter
23
 from operator import itemgetter
24
+from Queue import Queue
25
+from time import time
24
 from urlparse import urlparse, unquote
26
 from urlparse import urlparse, unquote
25
 from searx.engines import (
27
 from searx.engines import (
26
     categories, engines
28
     categories, engines
34
 
36
 
35
 
37
 
36
 def threaded_requests(requests):
38
 def threaded_requests(requests):
39
+    timeout_limit = max(r[2]['timeout'] for r in requests)
40
+    search_start = time()
37
     for fn, url, request_args in requests:
41
     for fn, url, request_args in requests:
38
         th = threading.Thread(
42
         th = threading.Thread(
39
             target=fn,
43
             target=fn,
45
 
49
 
46
     for th in threading.enumerate():
50
     for th in threading.enumerate():
47
         if th.name == 'search_request':
51
         if th.name == 'search_request':
48
-            th.join()
52
+            remaining_time = max(0.0, timeout_limit - (time() - search_start))
53
+            th.join(remaining_time)
54
+            if th.isAlive():
55
+                print('engine timeout')
56
+
49
 
57
 
50
 
58
 
51
 # get default reqest parameter
59
 # get default reqest parameter
56
 
64
 
57
 # create a callback wrapper for the search engine results
65
 # create a callback wrapper for the search engine results
58
 def make_callback(engine_name,
66
 def make_callback(engine_name,
59
-                  results,
67
+                  results_queue,
60
                   suggestions,
68
                   suggestions,
61
                   answers,
69
                   answers,
62
                   infoboxes,
70
                   infoboxes,
74
         except Exception, e:
82
         except Exception, e:
75
             # increase errors stats
83
             # increase errors stats
76
             engines[engine_name].stats['errors'] += 1
84
             engines[engine_name].stats['errors'] += 1
77
-            results[engine_name] = cb_res
85
+            results_queue.put_nowait((engine_name, cb_res))
78
 
86
 
79
             # print engine name and specific error message
87
             # print engine name and specific error message
80
             print '[E] Error with engine "{0}":\n\t{1}'.format(
88
             print '[E] Error with engine "{0}":\n\t{1}'.format(
104
             # append result
112
             # append result
105
             cb_res.append(result)
113
             cb_res.append(result)
106
 
114
 
107
-        results[engine_name] = cb_res
115
+        results_queue.put_nowait((engine_name, cb_res))
108
 
116
 
109
         # update stats with current page-load-time
117
         # update stats with current page-load-time
110
         engines[engine_name].stats['page_load_time'] += \
118
         engines[engine_name].stats['page_load_time'] += \
420
 
428
 
421
         # init vars
429
         # init vars
422
         requests = []
430
         requests = []
423
-        results = {}
431
+        results_queue = Queue()
424
         suggestions = set()
432
         suggestions = set()
425
         answers = set()
433
         answers = set()
426
         infoboxes = []
434
         infoboxes = []
468
             # create a callback wrapper for the search engine results
476
             # create a callback wrapper for the search engine results
469
             callback = make_callback(
477
             callback = make_callback(
470
                 selected_engine['name'],
478
                 selected_engine['name'],
471
-                results,
479
+                results_queue,
472
                 suggestions,
480
                 suggestions,
473
                 answers,
481
                 answers,
474
                 infoboxes,
482
                 infoboxes,
502
         # send all search-request
510
         # send all search-request
503
         threaded_requests(requests)
511
         threaded_requests(requests)
504
 
512
 
513
+        results = {}
514
+        while not results_queue.empty():
515
+            engine_name, engine_results = results_queue.get_nowait()
516
+            results[engine_name] = engine_results
517
+
505
         # update engine-specific stats
518
         # update engine-specific stats
506
         for engine_name, engine_results in results.items():
519
         for engine_name, engine_results in results.items():
507
             engines[engine_name].stats['search_count'] += 1
520
             engines[engine_name].stats['search_count'] += 1