Browse Source

[enh] engine cfg compatibilty

asciimoo 11 years ago
parent
commit
74b6be3991

+ 4
- 1
searx/engines/duckduckgo.py View File

1
 from json import loads
1
 from json import loads
2
+from urllib import urlencode
2
 
3
 
4
+url = 'https://duckduckgo.com/'
5
+search_url = url + 'd.js?{query}&l=us-en&p=1&s=0'
3
 
6
 
4
 def request(query, params):
7
 def request(query, params):
5
-    params['url'] = 'https://duckduckgo.com/d.js?q=%s&l=us-en&p=1&s=0' % query
8
+    params['url'] = search_url.format(query=urlencode({'q': query}))
6
     return params
9
     return params
7
 
10
 
8
 
11
 

+ 4
- 1
searx/engines/duckduckgo_definitions.py View File

1
 import json
1
 import json
2
+from urllib import urlencode
3
+
4
+url = 'http://api.duckduckgo.com/?{query}&format=json&pretty=0'
2
 
5
 
3
 def request(query, params):
6
 def request(query, params):
4
-    params['url'] = 'http://api.duckduckgo.com/?q=%s&format=json&pretty=0' % query
7
+    params['url'] =  url.format(query=urlencode({'q': query}))
5
     return params
8
     return params
6
 
9
 
7
 
10
 

+ 6
- 8
searx/engines/flickr.py View File

1
 #!/usr/bin/env python
1
 #!/usr/bin/env python
2
 
2
 
3
-from urllib import quote
3
+from urllib import urlencode
4
 from lxml import html
4
 from lxml import html
5
 from urlparse import urljoin
5
 from urlparse import urljoin
6
 
6
 
7
 categories = ['images']
7
 categories = ['images']
8
 
8
 
9
-base_url = 'https://secure.flickr.com/'
10
-search_url = base_url+'search/?q='
9
+url = 'https://secure.flickr.com/'
10
+search_url = url+'search/?q={query}'
11
 
11
 
12
 def request(query, params):
12
 def request(query, params):
13
-    global search_url
14
-    query = quote(query.replace(' ', '+'), safe='+')
15
-    params['url'] = search_url + query
13
+    params['url'] = search_url.format(query=urlencode({'q': query}))
16
     return params
14
     return params
17
 
15
 
18
 def response(resp):
16
 def response(resp):
20
     results = []
18
     results = []
21
     dom = html.fromstring(resp.text)
19
     dom = html.fromstring(resp.text)
22
     for result in dom.xpath('//div[@id="thumbnails"]//a[@class="rapidnofollow photo-click" and @data-track="photo-click"]'):
20
     for result in dom.xpath('//div[@id="thumbnails"]//a[@class="rapidnofollow photo-click" and @data-track="photo-click"]'):
23
-        url = urljoin(base_url, result.attrib.get('href'))
21
+        href = urljoin(url, result.attrib.get('href'))
24
         img = result.xpath('.//img')[0]
22
         img = result.xpath('.//img')[0]
25
         title = img.attrib.get('alt', '')
23
         title = img.attrib.get('alt', '')
26
         img_src = img.attrib.get('data-defer-src')
24
         img_src = img.attrib.get('data-defer-src')
27
         if not img_src:
25
         if not img_src:
28
             continue
26
             continue
29
-        results.append({'url': url, 'title': title, 'img_src': img_src, 'template': 'images.html'})
27
+        results.append({'url': href, 'title': title, 'img_src': img_src, 'template': 'images.html'})
30
     return results
28
     return results

+ 2
- 2
searx/engines/github.py View File

4
 
4
 
5
 categories = ['it']
5
 categories = ['it']
6
 
6
 
7
-search_url = 'https://api.github.com/search/repositories?sort=stars&order=desc&'
7
+search_url = 'https://api.github.com/search/repositories?sort=stars&order=desc&{query}'
8
 
8
 
9
 def request(query, params):
9
 def request(query, params):
10
     global search_url
10
     global search_url
11
-    params['url'] = search_url + urlencode({'q': query})
11
+    params['url'] = search_url.format(query=urlencode({'q': query}))
12
     params['headers']['Accept'] = 'application/vnd.github.preview.text-match+json'
12
     params['headers']['Accept'] = 'application/vnd.github.preview.text-match+json'
13
     return params
13
     return params
14
 
14
 

+ 4
- 5
searx/engines/google_images.py View File

5
 
5
 
6
 categories = ['images']
6
 categories = ['images']
7
 
7
 
8
-search_url = 'https://ajax.googleapis.com/ajax/services/search/images?v=1.0&start=0&rsz=large&safe=off&filter=off&'
8
+url = 'https://ajax.googleapis.com/'
9
+search_url = url + 'ajax/services/search/images?v=1.0&start=0&rsz=large&safe=off&filter=off&{query}'
9
 
10
 
10
 def request(query, params):
11
 def request(query, params):
11
-    global search_url
12
     params['url'] = search_url + urlencode({'q': query})
12
     params['url'] = search_url + urlencode({'q': query})
13
     return params
13
     return params
14
 
14
 
15
 def response(resp):
15
 def response(resp):
16
-    global base_url
17
     results = []
16
     results = []
18
     search_res = loads(resp.text)
17
     search_res = loads(resp.text)
19
     if not search_res.get('responseData'):
18
     if not search_res.get('responseData'):
21
     if not search_res['responseData'].get('results'):
20
     if not search_res['responseData'].get('results'):
22
         return []
21
         return []
23
     for result in search_res['responseData']['results']:
22
     for result in search_res['responseData']['results']:
24
-        url = result['originalContextUrl']
23
+        href = result['originalContextUrl']
25
         title = result['title']
24
         title = result['title']
26
         if not result['url']:
25
         if not result['url']:
27
             continue
26
             continue
28
-        results.append({'url': url, 'title': title, 'content': '', 'img_src': result['url'], 'template': 'images.html'})
27
+        results.append({'url': href, 'title': title, 'content': '', 'img_src': result['url'], 'template': 'images.html'})
29
     return results
28
     return results

+ 5
- 8
searx/engines/piratebay.py View File

5
 
5
 
6
 categories = ['videos', 'music']
6
 categories = ['videos', 'music']
7
 
7
 
8
-base_url = 'https://thepiratebay.sx/'
9
-search_url = base_url + 'search/{search_term}/0/99/{search_type}'
8
+url = 'https://thepiratebay.sx/'
9
+search_url = url + 'search/{search_term}/0/99/{search_type}'
10
 search_types = {'videos': '200'
10
 search_types = {'videos': '200'
11
                ,'music' : '100'
11
                ,'music' : '100'
12
                }
12
                }
13
 
13
 
14
 def request(query, params):
14
 def request(query, params):
15
-    global search_url, search_types
16
-    # 200 is the video category
17
     params['url'] = search_url.format(search_term=quote(query), search_type=search_types.get(params['category']))
15
     params['url'] = search_url.format(search_term=quote(query), search_type=search_types.get(params['category']))
18
     return params
16
     return params
19
 
17
 
20
 
18
 
21
 def response(resp):
19
 def response(resp):
22
-    global base_url
23
     results = []
20
     results = []
24
     dom = html.fromstring(resp.text)
21
     dom = html.fromstring(resp.text)
25
     search_res = dom.xpath('//table[@id="searchResult"]//tr')
22
     search_res = dom.xpath('//table[@id="searchResult"]//tr')
27
         return results
24
         return results
28
     for result in search_res[1:]:
25
     for result in search_res[1:]:
29
         link = result.xpath('.//div[@class="detName"]//a')[0]
26
         link = result.xpath('.//div[@class="detName"]//a')[0]
30
-        url = urljoin(base_url, link.attrib.get('href'))
27
+        href = urljoin(url, link.attrib.get('href'))
31
         title = ' '.join(link.xpath('.//text()'))
28
         title = ' '.join(link.xpath('.//text()'))
32
         content = escape(' '.join(result.xpath('.//font[@class="detDesc"]//text()')))
29
         content = escape(' '.join(result.xpath('.//font[@class="detDesc"]//text()')))
33
         seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]
30
         seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]
34
         content += '<br />Seed: %s, Leech: %s' % (seed, leech)
31
         content += '<br />Seed: %s, Leech: %s' % (seed, leech)
35
         magnetlink = result.xpath('.//a[@title="Download this torrent using magnet"]')[0]
32
         magnetlink = result.xpath('.//a[@title="Download this torrent using magnet"]')[0]
36
-        content += '<br /><a href="%s">magnet link</a>' % urljoin(base_url, magnetlink.attrib['href'])
37
-        results.append({'url': url, 'title': title, 'content': content})
33
+        content += '<br /><a href="%s">magnet link</a>' % urljoin(url, magnetlink.attrib['href'])
34
+        results.append({'url': href, 'title': title, 'content': content})
38
     return results
35
     return results

+ 4
- 2
searx/engines/soundcloud.py View File

1
 from json import loads
1
 from json import loads
2
+from urllib import urlencode
2
 
3
 
3
 categories = ['music']
4
 categories = ['music']
4
 
5
 
5
 guest_client_id = 'b45b1aa10f1ac2941910a7f0d10f8e28'
6
 guest_client_id = 'b45b1aa10f1ac2941910a7f0d10f8e28'
6
-search_url = 'https://api.soundcloud.com/search?q=%s&facet=model&limit=10&offset=0&linked_partitioning=1&client_id='+guest_client_id
7
+url = 'https://api.soundcloud.com/'
8
+search_url = url + 'search?{query}&facet=model&limit=20&offset=0&linked_partitioning=1&client_id='+guest_client_id
7
 
9
 
8
 def request(query, params):
10
 def request(query, params):
9
     global search_url
11
     global search_url
10
-    params['url'] = search_url % query
12
+    params['url'] = search_url.format(query=urlencode({'q': query}))
11
     return params
13
     return params
12
 
14
 
13
 
15
 

+ 4
- 6
searx/engines/stackoverflow.py View File

5
 
5
 
6
 categories = ['it']
6
 categories = ['it']
7
 
7
 
8
-base_url = 'http://stackoverflow.com/'
9
-search_url = base_url+'search?'
8
+url = 'http://stackoverflow.com/'
9
+search_url = url+'search?'
10
 
10
 
11
 def request(query, params):
11
 def request(query, params):
12
-    global search_url
13
     params['url'] = search_url + urlencode({'q': query})
12
     params['url'] = search_url + urlencode({'q': query})
14
     return params
13
     return params
15
 
14
 
16
 
15
 
17
 def response(resp):
16
 def response(resp):
18
-    global base_url
19
     results = []
17
     results = []
20
     dom = html.fromstring(resp.text)
18
     dom = html.fromstring(resp.text)
21
     for result in dom.xpath('//div[@class="question-summary search-result"]'):
19
     for result in dom.xpath('//div[@class="question-summary search-result"]'):
22
         link = result.xpath('.//div[@class="result-link"]//a')[0]
20
         link = result.xpath('.//div[@class="result-link"]//a')[0]
23
-        url = urljoin(base_url, link.attrib.get('href'))
21
+        href = urljoin(url, link.attrib.get('href'))
24
         title = ' '.join(link.xpath('.//text()'))
22
         title = ' '.join(link.xpath('.//text()'))
25
         content = escape(' '.join(result.xpath('.//div[@class="excerpt"]//text()')))
23
         content = escape(' '.join(result.xpath('.//div[@class="excerpt"]//text()')))
26
-        results.append({'url': url, 'title': title, 'content': content})
24
+        results.append({'url': href, 'title': title, 'content': content})
27
     return results
25
     return results

+ 3
- 6
searx/engines/youtube.py View File

1
 from json import loads
1
 from json import loads
2
-from urllib import quote
2
+from urllib import urlencode
3
 
3
 
4
 categories = ['videos']
4
 categories = ['videos']
5
 
5
 
6
-search_url = 'https://gdata.youtube.com/feeds/api/videos?alt=json&q='
6
+search_url = 'https://gdata.youtube.com/feeds/api/videos?alt=json&{query}'
7
 
7
 
8
 def request(query, params):
8
 def request(query, params):
9
-    global search_url
10
-    query = quote(query.replace(' ', '+'), safe='+')
11
-    params['url'] = search_url + query
12
-
9
+    params['url'] = search_url.format(query=urlencode({'q': query}))
13
     return params
10
     return params
14
 
11
 
15
 
12