free
/
searx
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 11 Wiki Activity
Browse Source

Add F-Droid search engine

Kirill Isakov 9 years ago
parent
547b8a8765
commit
c1d456b136
3 changed files with 107 additions and 0 deletions
Split View Show Diff Stats
  1. 53
    0
      searx/engines/fdroid.py
  2. 5
    0
      searx/settings.yml
  3. 49
    0
      tests/unit/engines/test_fdroid.py

+ 53
- 0
searx/engines/fdroid.py View File 

@@ -0,0 +1,53 @@
1 
+"""
2 
+ F-Droid (a repository of FOSS applications for Android)
3 
+
4 
+ @website      https://f-droid.org/
5 
+ @provide-api  no
6 
+ @using-api    no
7 
+ @results      HTML
8 
+ @stable       no (HTML can change)
9 
+ @parse        url, title, content
10 
+"""
11 
+
12 
+from cgi import escape
13 
+from urllib import urlencode
14 
+from searx.engines.xpath import extract_text
15 
+from lxml import html
16 
+
17 
+# engine dependent config
18 
+categories = ['files']
19 
+paging = True
20 
+
21 
+# search-url
22 
+base_url = 'https://f-droid.org/'
23 
+search_url = base_url + 'repository/browse/?{query}'
24 
+
25 
+
26 
+# do search-request
27 
+def request(query, params):
28 
+    query = urlencode({'fdfilter': query,
29 
+                       'fdpage': params['pageno']})
30 
+    params['url'] = search_url.format(query=query)
31 
+    return params
32 
+
33 
+
34 
+# get response from search-request
35 
+def response(resp):
36 
+    results = []
37 
+
38 
+    dom = html.fromstring(resp.text)
39 
+
40 
+    for app in dom.xpath('//div[@id="appheader"]'):
41 
+        url = app.xpath('./ancestor::a/@href')[0]
42 
+        title = app.xpath('./p/span/text()')[0]
43 
+        img_src = app.xpath('.//img/@src')[0]
44 
+
45 
+        content = extract_text(app.xpath('./p')[0])
46 
+        content = escape(content.replace(title, '', 1).strip())
47 
+
48 
+        results.append({'url': url,
49 
+                        'title': title,
50 
+                        'content': content,
51 
+                        'img_src': img_src})
52 
+
53 
+    return results

+ 5
- 0
searx/settings.yml View File 

@@ -105,6 +105,11 @@ engines:
105 105 
     shortcut : 1x
106 106 
     disabled : True
107 107
108 
+  - name : fdroid
109 
+    engine : fdroid
110 
+    shortcut : fd
111 
+    disabled : True
112 
+
108 113 
   - name : flickr
109 114 
     categories : images
110 115 
     shortcut : fl

+ 49
- 0
tests/unit/engines/test_fdroid.py View File 

@@ -0,0 +1,49 @@
1 
+import mock
2 
+from collections import defaultdict
3 
+from searx.engines import fdroid
4 
+from searx.testing import SearxTestCase
5 
+
6 
+
7 
+class TestFdroidEngine(SearxTestCase):
8 
+
9 
+    def test_request(self):
10 
+        query = 'test_query'
11 
+        dic = defaultdict(dict)
12 
+        dic['pageno'] = 1
13 
+        params = fdroid.request(query, dic)
14 
+        self.assertTrue('url' in params)
15 
+        self.assertTrue(query in params['url'])
16 
+        self.assertTrue('f-droid.org' in params['url'])
17 
+
18 
+    def test_response(self):
19 
+        resp = mock.Mock(text='<html></html>')
20 
+        self.assertEqual(fdroid.response(resp), [])
21 
+
22 
+        html = """
23 
+        <a href="https://google.com/qwerty">
24 
+          <div id="appheader">
25 
+            <div style="float:left;padding-right:10px;">
26 
+              <img src="http://example.com/image.png"
27 
+                   style="width:48px;border:none;">
28 
+            </div>
29 
+            <div style="float:right;">
30 
+              <p>Details...</p>
31 
+            </div>
32 
+            <p style="color:#000000;">
33 
+              <span style="font-size:20px;">Sample title</span>
34 
+              <br>
35 
+              Sample content
36 
+            </p>
37 
+          </div>
38 
+        </a>
39 
+        """
40 
+
41 
+        resp = mock.Mock(text=html)
42 
+        results = fdroid.response(resp)
43 
+
44 
+        self.assertEqual(type(results), list)
45 
+        self.assertEqual(len(results), 1)
46 
+        self.assertEqual(results[0]['url'], 'https://google.com/qwerty')
47 
+        self.assertEqual(results[0]['title'], 'Sample title')
48 
+        self.assertEqual(results[0]['content'], 'Sample content')
49 
+        self.assertEqual(results[0]['img_src'], 'http://example.com/image.png')