|
@@ -47,16 +47,12 @@ logger = logger.getChild('search')
|
47
|
47
|
number_of_searches = 0
|
48
|
48
|
|
49
|
49
|
|
50
|
|
-def send_http_request(engine, request_params, start_time, timeout_limit):
|
51
|
|
- # for page_load_time stats
|
52
|
|
- time_before_request = time()
|
53
|
|
-
|
|
50
|
+def send_http_request(engine, request_params):
|
54
|
51
|
# create dictionary which contain all
|
55
|
52
|
# informations about the request
|
56
|
53
|
request_args = dict(
|
57
|
54
|
headers=request_params['headers'],
|
58
|
55
|
cookies=request_params['cookies'],
|
59
|
|
- timeout=timeout_limit,
|
60
|
56
|
verify=request_params['verify']
|
61
|
57
|
)
|
62
|
58
|
|
|
@@ -68,29 +64,10 @@ def send_http_request(engine, request_params, start_time, timeout_limit):
|
68
|
64
|
request_args['data'] = request_params['data']
|
69
|
65
|
|
70
|
66
|
# send the request
|
71
|
|
- response = req(request_params['url'], **request_args)
|
72
|
|
-
|
73
|
|
- # is there a timeout (no parsing in this case)
|
74
|
|
- timeout_overhead = 0.2 # seconds
|
75
|
|
- time_after_request = time()
|
76
|
|
- search_duration = time_after_request - start_time
|
77
|
|
- if search_duration > timeout_limit + timeout_overhead:
|
78
|
|
- raise requests.exceptions.Timeout(response=response)
|
79
|
|
-
|
80
|
|
- with threading.RLock():
|
81
|
|
- # no error : reset the suspend variables
|
82
|
|
- engine.continuous_errors = 0
|
83
|
|
- engine.suspend_end_time = 0
|
84
|
|
- # update stats with current page-load-time
|
85
|
|
- # only the HTTP request
|
86
|
|
- engine.stats['page_load_time'] += time_after_request - time_before_request
|
87
|
|
- engine.stats['page_load_count'] += 1
|
|
67
|
+ return req(request_params['url'], **request_args)
|
88
|
68
|
|
89
|
|
- # everything is ok : return the response
|
90
|
|
- return response
|
91
|
69
|
|
92
|
|
-
|
93
|
|
-def search_one_request(engine, query, request_params, start_time, timeout_limit):
|
|
70
|
+def search_one_request(engine, query, request_params):
|
94
|
71
|
# update request parameters dependent on
|
95
|
72
|
# search-engine (contained in engines folder)
|
96
|
73
|
engine.request(query, request_params)
|
|
@@ -103,7 +80,7 @@ def search_one_request(engine, query, request_params, start_time, timeout_limit)
|
103
|
80
|
return []
|
104
|
81
|
|
105
|
82
|
# send request
|
106
|
|
- response = send_http_request(engine, request_params, start_time, timeout_limit)
|
|
83
|
+ response = send_http_request(engine, request_params)
|
107
|
84
|
|
108
|
85
|
# parse the response
|
109
|
86
|
response.search_params = request_params
|
|
@@ -111,11 +88,20 @@ def search_one_request(engine, query, request_params, start_time, timeout_limit)
|
111
|
88
|
|
112
|
89
|
|
113
|
90
|
def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
|
|
91
|
+ # set timeout for all HTTP requests
|
|
92
|
+ requests_lib.set_timeout_for_thread(timeout_limit, start_time=start_time)
|
|
93
|
+ # reset the HTTP total time
|
|
94
|
+ requests_lib.reset_time_for_thread()
|
|
95
|
+
|
|
96
|
+ #
|
114
|
97
|
engine = engines[engine_name]
|
115
|
98
|
|
|
99
|
+ # suppose everything will be alright
|
|
100
|
+ requests_exception = False
|
|
101
|
+
|
116
|
102
|
try:
|
117
|
103
|
# send requests and parse the results
|
118
|
|
- search_results = search_one_request(engine, query, request_params, start_time, timeout_limit)
|
|
104
|
+ search_results = search_one_request(engine, query, request_params)
|
119
|
105
|
|
120
|
106
|
# add results
|
121
|
107
|
result_container.extend(engine_name, search_results)
|
|
@@ -124,14 +110,15 @@ def search_one_request_safe(engine_name, query, request_params, result_container
|
124
|
110
|
with threading.RLock():
|
125
|
111
|
engine.stats['engine_time'] += time() - start_time
|
126
|
112
|
engine.stats['engine_time_count'] += 1
|
127
|
|
-
|
128
|
|
- return True
|
|
113
|
+ # update stats with the total HTTP time
|
|
114
|
+ engine.stats['page_load_time'] += requests_lib.get_time_for_thread()
|
|
115
|
+ engine.stats['page_load_count'] += 1
|
129
|
116
|
|
130
|
117
|
except Exception as e:
|
131
|
|
- engine.stats['errors'] += 1
|
132
|
|
-
|
133
|
118
|
search_duration = time() - start_time
|
134
|
|
- requests_exception = False
|
|
119
|
+
|
|
120
|
+ with threading.RLock():
|
|
121
|
+ engine.stats['errors'] += 1
|
135
|
122
|
|
136
|
123
|
if (issubclass(e.__class__, requests.exceptions.Timeout)):
|
137
|
124
|
result_container.add_unresponsive_engine((engine_name, gettext('timeout')))
|
|
@@ -152,14 +139,17 @@ def search_one_request_safe(engine_name, query, request_params, result_container
|
152
|
139
|
# others errors
|
153
|
140
|
logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
|
154
|
141
|
|
155
|
|
- # update continuous_errors / suspend_end_time
|
|
142
|
+ # suspend or not the engine if there are HTTP errors
|
|
143
|
+ with threading.RLock():
|
156
|
144
|
if requests_exception:
|
157
|
|
- with threading.RLock():
|
158
|
|
- engine.continuous_errors += 1
|
159
|
|
- engine.suspend_end_time = time() + min(60, engine.continuous_errors)
|
160
|
|
-
|
161
|
|
- #
|
162
|
|
- return False
|
|
145
|
+ # update continuous_errors / suspend_end_time
|
|
146
|
+ engine.continuous_errors += 1
|
|
147
|
+ engine.suspend_end_time = time() + min(60, engine.continuous_errors)
|
|
148
|
+ else:
|
|
149
|
+ # no HTTP error (perhaps an engine error)
|
|
150
|
+ # anyway, reset the suspend variables
|
|
151
|
+ engine.continuous_errors = 0
|
|
152
|
+ engine.suspend_end_time = 0
|
163
|
153
|
|
164
|
154
|
|
165
|
155
|
def search_multiple_requests(requests, result_container, start_time, timeout_limit):
|