|
@@ -6,20 +6,25 @@ search_url = None
|
6
|
6
|
|
7
|
7
|
# TODO paging
|
8
|
8
|
paging = False
|
|
9
|
+# TODO complete list of country mapping
|
|
10
|
+country_map = {'en_US': 'eng',
|
|
11
|
+ 'en_UK': 'uk',
|
|
12
|
+ 'nl_NL': 'ned'}
|
9
|
13
|
|
10
|
14
|
|
11
|
15
|
def request(query, params):
|
12
|
|
- global search_url
|
13
|
16
|
query = urlencode({'q': query})[2:]
|
14
|
17
|
params['url'] = search_url
|
15
|
18
|
params['method'] = 'POST'
|
16
|
19
|
params['data'] = {'query': query,
|
17
|
20
|
'startat': (params['pageno'] - 1) * 10} # offset
|
|
21
|
+ country = country_map.get(params['language'], 'eng')
|
|
22
|
+ params['cookies']['preferences'] = \
|
|
23
|
+ 'lang_homepageEEEs/air/{country}/N1NsslEEE1N1Nfont_sizeEEEmediumN1Nrecent_results_filterEEE1N1Nlanguage_uiEEEenglishN1Ndisable_open_in_new_windowEEE0N1Ncolor_schemeEEEnewN1Nnum_of_resultsEEE10N1N'.format(country=country) # noqa
|
18
|
24
|
return params
|
19
|
25
|
|
20
|
26
|
|
21
|
27
|
def response(resp):
|
22
|
|
- global base_url
|
23
|
28
|
results = []
|
24
|
29
|
dom = html.fromstring(resp.content)
|
25
|
30
|
# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
|