|
@@ -37,7 +37,7 @@ logger = logger.getChild('search')
|
37
|
37
|
number_of_searches = 0
|
38
|
38
|
|
39
|
39
|
|
40
|
|
-def send_http_request(engine, request_params, timeout_limit):
|
|
40
|
+def send_http_request(engine, request_params, start_time, timeout_limit):
|
41
|
41
|
# for page_load_time stats
|
42
|
42
|
time_before_request = time()
|
43
|
43
|
|
|
@@ -62,9 +62,10 @@ def send_http_request(engine, request_params, timeout_limit):
|
62
|
62
|
|
63
|
63
|
# is there a timeout (no parsing in this case)
|
64
|
64
|
timeout_overhead = 0.2 # seconds
|
65
|
|
- search_duration = time() - request_params['started']
|
|
65
|
+ time_after_request = time()
|
|
66
|
+ search_duration = time_after_request - start_time
|
66
|
67
|
if search_duration > timeout_limit + timeout_overhead:
|
67
|
|
- raise Timeout(response=response)
|
|
68
|
+ raise requests.exceptions.Timeout(response=response)
|
68
|
69
|
|
69
|
70
|
with threading.RLock():
|
70
|
71
|
# no error : reset the suspend variables
|
|
@@ -72,14 +73,14 @@ def send_http_request(engine, request_params, timeout_limit):
|
72
|
73
|
engine.suspend_end_time = 0
|
73
|
74
|
# update stats with current page-load-time
|
74
|
75
|
# only the HTTP request
|
75
|
|
- engine.stats['page_load_time'] += time() - time_before_request
|
|
76
|
+ engine.stats['page_load_time'] += time_after_request - time_before_request
|
76
|
77
|
engine.stats['page_load_count'] += 1
|
77
|
78
|
|
78
|
79
|
# everything is ok : return the response
|
79
|
80
|
return response
|
80
|
81
|
|
81
|
82
|
|
82
|
|
-def search_one_request(engine, query, request_params, timeout_limit):
|
|
83
|
+def search_one_request(engine, query, request_params, start_time, timeout_limit):
|
83
|
84
|
# update request parameters dependent on
|
84
|
85
|
# search-engine (contained in engines folder)
|
85
|
86
|
engine.request(query, request_params)
|
|
@@ -92,24 +93,21 @@ def search_one_request(engine, query, request_params, timeout_limit):
|
92
|
93
|
return []
|
93
|
94
|
|
94
|
95
|
# send request
|
95
|
|
- response = send_http_request(engine, request_params, timeout_limit)
|
|
96
|
+ response = send_http_request(engine, request_params, start_time, timeout_limit)
|
96
|
97
|
|
97
|
98
|
# parse the response
|
98
|
99
|
response.search_params = request_params
|
99
|
100
|
return engine.response(response)
|
100
|
101
|
|
101
|
102
|
|
102
|
|
-def search_one_request_safe(engine_name, query, request_params, result_container, timeout_limit):
|
103
|
|
- start_time = time()
|
|
103
|
+def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
|
104
|
104
|
engine = engines[engine_name]
|
105
|
105
|
|
106
|
106
|
try:
|
107
|
107
|
# send requests and parse the results
|
108
|
|
- search_results = search_one_request(engine, query, request_params, timeout_limit)
|
|
108
|
+ search_results = search_one_request(engine, query, request_params, start_time, timeout_limit)
|
109
|
109
|
|
110
|
110
|
# add results
|
111
|
|
- for result in search_results:
|
112
|
|
- result['engine'] = engine_name
|
113
|
111
|
result_container.extend(engine_name, search_results)
|
114
|
112
|
|
115
|
113
|
# update engine time when there is no exception
|
|
@@ -131,7 +129,7 @@ def search_one_request_safe(engine_name, query, request_params, result_container
|
131
|
129
|
"(search duration : {1} s, timeout: {2} s) : {3}"
|
132
|
130
|
.format(engine_name, search_duration, timeout_limit, e.__class__.__name__))
|
133
|
131
|
requests_exception = True
|
134
|
|
- if (issubclass(e.__class__, requests.exceptions.RequestException)):
|
|
132
|
+ elif (issubclass(e.__class__, requests.exceptions.RequestException)):
|
135
|
133
|
# other requests exception
|
136
|
134
|
logger.exception("engine {0} : requests exception"
|
137
|
135
|
"(search duration : {1} s, timeout: {2} s) : {3}"
|
|
@@ -151,14 +149,13 @@ def search_one_request_safe(engine_name, query, request_params, result_container
|
151
|
149
|
return False
|
152
|
150
|
|
153
|
151
|
|
154
|
|
-def search_multiple_requests(requests, result_container, timeout_limit):
|
155
|
|
- start_time = time()
|
|
152
|
+def search_multiple_requests(requests, result_container, start_time, timeout_limit):
|
156
|
153
|
search_id = uuid4().__str__()
|
157
|
154
|
|
158
|
155
|
for engine_name, query, request_params in requests:
|
159
|
156
|
th = threading.Thread(
|
160
|
157
|
target=search_one_request_safe,
|
161
|
|
- args=(engine_name, query, request_params, result_container, timeout_limit),
|
|
158
|
+ args=(engine_name, query, request_params, result_container, start_time, timeout_limit),
|
162
|
159
|
name=search_id,
|
163
|
160
|
)
|
164
|
161
|
th._engine_name = engine_name
|
|
@@ -368,7 +365,6 @@ class Search(object):
|
368
|
365
|
request_params = default_request_params()
|
369
|
366
|
request_params['headers']['User-Agent'] = user_agent
|
370
|
367
|
request_params['category'] = selected_engine['category']
|
371
|
|
- request_params['started'] = start_time
|
372
|
368
|
request_params['pageno'] = search_query.pageno
|
373
|
369
|
|
374
|
370
|
if hasattr(engine, 'language') and engine.language:
|
|
@@ -388,7 +384,7 @@ class Search(object):
|
388
|
384
|
|
389
|
385
|
if requests:
|
390
|
386
|
# send all search-request
|
391
|
|
- search_multiple_requests(requests, self.result_container, timeout_limit - (time() - start_time))
|
|
387
|
+ search_multiple_requests(requests, self.result_container, start_time, timeout_limit)
|
392
|
388
|
start_new_thread(gc.collect, tuple())
|
393
|
389
|
|
394
|
390
|
# return results, suggestions, answers and infoboxes
|