Browse Source

[enh] startpage engine added

asciimoo 11 years ago
parent
commit
57eaeb9b74
1 changed files with 28 additions and 0 deletions
  1. 28
    0
      searx/engines/startpage.py

+ 28
- 0
searx/engines/startpage.py View File

1
+from urllib import quote
2
+from lxml import html
3
+from urlparse import urljoin
4
+from cgi import escape
5
+
6
+base_url = 'http://startpage.com/'
7
+search_url = base_url+'do/search'
8
+
9
+def request(query, params):
10
+    global search_url
11
+    query = quote(query.replace(' ', '+'), safe='+')
12
+    params['url'] = search_url
13
+    params['method'] = 'POST'
14
+    params['data'] = {'query': query}
15
+    return params
16
+
17
+
18
+def response(resp):
19
+    global base_url
20
+    results = []
21
+    dom = html.fromstring(resp.text)
22
+    for result in dom.xpath('//div[@class="result"]'):
23
+        link = result.xpath('.//h3/a')[0]
24
+        url = urljoin(base_url, link.attrib.get('href'))
25
+        title = ' '.join(link.xpath('.//text()'))
26
+        content = escape(' '.join(result.xpath('.//p[@class="desc"]//text()')))
27
+        results.append({'url': url, 'title': title, 'content': content})
28
+    return results