浏览代码

[mod] timeout in log are readable (the timeouts are compare to the start_time of the request).

Alexandre Flament 8 年前
父节点
当前提交
28d51fd063
共有 1 个文件被更改,包括 11 次插入13 次删除
  1. 11
    13
      searx/search.py

+ 11
- 13
searx/search.py 查看文件

@@ -37,7 +37,7 @@ logger = logger.getChild('search')
37 37
 number_of_searches = 0
38 38
 
39 39
 
40
-def send_http_request(engine, request_params, timeout_limit):
40
+def send_http_request(engine, request_params, start_time, timeout_limit):
41 41
     # for page_load_time stats
42 42
     time_before_request = time()
43 43
 
@@ -62,7 +62,8 @@ def send_http_request(engine, request_params, timeout_limit):
62 62
 
63 63
     # is there a timeout (no parsing in this case)
64 64
     timeout_overhead = 0.2  # seconds
65
-    search_duration = time() - request_params['started']
65
+    time_after_request = time()
66
+    search_duration = time_after_request - start_time
66 67
     if search_duration > timeout_limit + timeout_overhead:
67 68
         raise Timeout(response=response)
68 69
 
@@ -72,14 +73,14 @@ def send_http_request(engine, request_params, timeout_limit):
72 73
         engine.suspend_end_time = 0
73 74
         # update stats with current page-load-time
74 75
         # only the HTTP request
75
-        engine.stats['page_load_time'] += time() - time_before_request
76
+        engine.stats['page_load_time'] += time_after_request - time_before_request
76 77
         engine.stats['page_load_count'] += 1
77 78
 
78 79
     # everything is ok : return the response
79 80
     return response
80 81
 
81 82
 
82
-def search_one_request(engine, query, request_params, timeout_limit):
83
+def search_one_request(engine, query, request_params, start_time, timeout_limit):
83 84
     # update request parameters dependent on
84 85
     # search-engine (contained in engines folder)
85 86
     engine.request(query, request_params)
@@ -92,20 +93,19 @@ def search_one_request(engine, query, request_params, timeout_limit):
92 93
         return []
93 94
 
94 95
     # send request
95
-    response = send_http_request(engine, request_params, timeout_limit)
96
+    response = send_http_request(engine, request_params, start_time, timeout_limit)
96 97
 
97 98
     # parse the response
98 99
     response.search_params = request_params
99 100
     return engine.response(response)
100 101
 
101 102
 
102
-def search_one_request_safe(engine_name, query, request_params, result_container, timeout_limit):
103
-    start_time = time()
103
+def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
104 104
     engine = engines[engine_name]
105 105
 
106 106
     try:
107 107
         # send requests and parse the results
108
-        search_results = search_one_request(engine, query, request_params, timeout_limit)
108
+        search_results = search_one_request(engine, query, request_params, start_time, timeout_limit)
109 109
 
110 110
         # add results
111 111
         result_container.extend(engine_name, search_results)
@@ -149,14 +149,13 @@ def search_one_request_safe(engine_name, query, request_params, result_container
149 149
         return False
150 150
 
151 151
 
152
-def search_multiple_requests(requests, result_container, timeout_limit):
153
-    start_time = time()
152
+def search_multiple_requests(requests, result_container, start_time, timeout_limit):
154 153
     search_id = uuid4().__str__()
155 154
 
156 155
     for engine_name, query, request_params in requests:
157 156
         th = threading.Thread(
158 157
             target=search_one_request_safe,
159
-            args=(engine_name, query, request_params, result_container, timeout_limit),
158
+            args=(engine_name, query, request_params, result_container, start_time, timeout_limit),
160 159
             name=search_id,
161 160
         )
162 161
         th._engine_name = engine_name
@@ -366,7 +365,6 @@ class Search(object):
366 365
             request_params = default_request_params()
367 366
             request_params['headers']['User-Agent'] = user_agent
368 367
             request_params['category'] = selected_engine['category']
369
-            request_params['started'] = start_time
370 368
             request_params['pageno'] = search_query.pageno
371 369
 
372 370
             if hasattr(engine, 'language') and engine.language:
@@ -386,7 +384,7 @@ class Search(object):
386 384
 
387 385
         if requests:
388 386
             # send all search-request
389
-            search_multiple_requests(requests, self.result_container, timeout_limit - (time() - start_time))
387
+            search_multiple_requests(requests, self.result_container, start_time, timeout_limit)
390 388
             start_new_thread(gc.collect, tuple())
391 389
 
392 390
         # return results, suggestions, answers and infoboxes