|
@@ -37,7 +37,7 @@ def extract_text(xpath_results):
|
37
|
37
|
return html_to_text(xpath_results.text_content())
|
38
|
38
|
|
39
|
39
|
|
40
|
|
-def extract_url(xpath_results, search_url=search_url):
|
|
40
|
+def extract_url(xpath_results, search_url):
|
41
|
41
|
url = extract_text(xpath_results)
|
42
|
42
|
|
43
|
43
|
if url.startswith('//'):
|
|
@@ -86,7 +86,7 @@ def response(resp):
|
86
|
86
|
dom = html.fromstring(resp.text)
|
87
|
87
|
if results_xpath:
|
88
|
88
|
for result in dom.xpath(results_xpath):
|
89
|
|
- url = extract_url(result.xpath(url_xpath))
|
|
89
|
+ url = extract_url(result.xpath(url_xpath), search_url)
|
90
|
90
|
title = extract_text(result.xpath(title_xpath)[0])
|
91
|
91
|
content = extract_text(result.xpath(content_xpath)[0])
|
92
|
92
|
results.append({'url': url, 'title': title, 'content': content})
|