|
@@ -104,36 +104,40 @@ def search(query, request, selected_categories):
|
104
|
104
|
selected_engines = []
|
105
|
105
|
number_of_searches += 1
|
106
|
106
|
user_agent = request.headers.get('User-Agent', '')
|
|
107
|
+
|
107
|
108
|
if not len(selected_categories):
|
108
|
109
|
selected_categories = ['general']
|
|
110
|
+
|
109
|
111
|
for categ in selected_categories:
|
110
|
112
|
selected_engines.extend({'category': categ, 'name': x.name} for x in categories[categ])
|
|
113
|
+
|
111
|
114
|
for selected_engine in selected_engines:
|
112
|
115
|
if selected_engine['name'] not in engines:
|
113
|
116
|
continue
|
|
117
|
+
|
114
|
118
|
engine = engines[selected_engine['name']]
|
|
119
|
+
|
115
|
120
|
request_params = default_request_params()
|
116
|
121
|
request_params['headers']['User-Agent'] = user_agent
|
117
|
122
|
request_params['category'] = selected_engine['category']
|
118
|
123
|
request_params['started'] = datetime.now()
|
119
|
124
|
request_params = engine.request(query, request_params)
|
|
125
|
+
|
120
|
126
|
callback = make_callback(selected_engine['name'], results, engine.response, request_params)
|
|
127
|
+
|
|
128
|
+ request_args = dict(headers = request_params['headers']
|
|
129
|
+ ,hooks = dict(response=callback)
|
|
130
|
+ ,cookies = request_params['cookies']
|
|
131
|
+ ,timeout = settings.request_timeout
|
|
132
|
+ )
|
|
133
|
+
|
121
|
134
|
if request_params['method'] == 'GET':
|
122
|
|
- req = grequests.get(request_params['url']
|
123
|
|
- ,headers = request_params['headers']
|
124
|
|
- ,hooks = dict(response=callback)
|
125
|
|
- ,cookies = request_params['cookies']
|
126
|
|
- ,timeout = settings.request_timeout
|
127
|
|
- )
|
|
135
|
+ req = grequests.get
|
128
|
136
|
else:
|
129
|
|
- req = grequests.post(request_params['url']
|
130
|
|
- ,data = request_params['data']
|
131
|
|
- ,headers = request_params['headers']
|
132
|
|
- ,hooks = dict(response=callback)
|
133
|
|
- ,cookies = request_params['cookies']
|
134
|
|
- ,timeout = settings.request_timeout
|
135
|
|
- )
|
136
|
|
- requests.append(req)
|
|
137
|
+ req = grequests.post
|
|
138
|
+ request_args['data'] = request_params['data']
|
|
139
|
+
|
|
140
|
+ requests.append(req(request_params['url'], **request_args))
|
137
|
141
|
grequests.map(requests)
|
138
|
142
|
for engine_name,engine_results in results.items():
|
139
|
143
|
engines[engine_name].stats['search_count'] += 1
|