Procházet zdrojové kódy

Merge pull request #746 from kvch/moar-time-range-support

Support time range search in more engines
Adam Tauber před 8 roky
rodič
revize
e23c8f954b

+ 7
- 0
searx/engines/bing_images.py Zobrazit soubor

@@ -24,11 +24,16 @@ import re
24 24
 categories = ['images']
25 25
 paging = True
26 26
 safesearch = True
27
+time_range_support = True
27 28
 
28 29
 # search-url
29 30
 base_url = 'https://www.bing.com/'
30 31
 search_string = 'images/search?{query}&count=10&first={offset}'
32
+time_range_string = '&qft=+filterui:age-lt{interval}'
31 33
 thumb_url = "https://www.bing.com/th?id={ihk}"
34
+time_range_dict = {'day': '1440',
35
+                   'week': '10080',
36
+                   'month': '43200'}
32 37
 
33 38
 # safesearch definitions
34 39
 safesearch_types = {2: 'STRICT',
@@ -58,6 +63,8 @@ def request(query, params):
58 63
         '&ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
59 64
 
60 65
     params['url'] = base_url + search_path
66
+    if params['time_range'] in time_range_dict:
67
+        params['url'] += time_range_string.format(interval=time_range_dict[params['time_range']])
61 68
 
62 69
     return params
63 70
 

+ 19
- 5
searx/engines/bing_news.py Zobrazit soubor

@@ -22,10 +22,15 @@ from searx.utils import list_get
22 22
 categories = ['news']
23 23
 paging = True
24 24
 language_support = True
25
+time_range_support = True
25 26
 
26 27
 # search-url
27 28
 base_url = 'https://www.bing.com/'
28 29
 search_string = 'news/search?{query}&first={offset}&format=RSS'
30
+search_string_with_time = 'news/search?{query}&first={offset}&qft=interval%3d"{interval}"&format=RSS'
31
+time_range_dict = {'day': '7',
32
+                   'week': '8',
33
+                   'month': '9'}
29 34
 
30 35
 
31 36
 # remove click
@@ -46,6 +51,19 @@ def image_url_cleanup(url_string):
46 51
     return url_string
47 52
 
48 53
 
54
+def _get_url(query, language, offset, time_range):
55
+    if time_range in time_range_dict:
56
+        search_path = search_string_with_time.format(
57
+            query=urlencode({'q': query, 'setmkt': language}),
58
+            offset=offset,
59
+            interval=time_range_dict[time_range])
60
+    else:
61
+        search_path = search_string.format(
62
+            query=urlencode({'q': query, 'setmkt': language}),
63
+            offset=offset)
64
+    return base_url + search_path
65
+
66
+
49 67
 # do search-request
50 68
 def request(query, params):
51 69
     offset = (params['pageno'] - 1) * 10 + 1
@@ -55,11 +73,7 @@ def request(query, params):
55 73
     else:
56 74
         language = params['language'].replace('_', '-')
57 75
 
58
-    search_path = search_string.format(
59
-        query=urlencode({'q': query, 'setmkt': language}),
60
-        offset=offset)
61
-
62
-    params['url'] = base_url + search_path
76
+    params['url'] = _get_url(query, language, offset, params['time_range'])
63 77
 
64 78
     return params
65 79
 

+ 14
- 3
searx/engines/flickr_noapi.py Zobrazit soubor

@@ -14,6 +14,7 @@
14 14
 
15 15
 from urllib import urlencode
16 16
 from json import loads
17
+from time import time
17 18
 import re
18 19
 from searx.engines import logger
19 20
 
@@ -24,21 +25,31 @@ categories = ['images']
24 25
 
25 26
 url = 'https://www.flickr.com/'
26 27
 search_url = url + 'search?{query}&page={page}'
28
+time_range_url = '&min_upload_date={start}&max_upload_date={end}'
27 29
 photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
28 30
 regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)
29 31
 image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
30 32
 
31 33
 paging = True
34
+time_range_support = True
35
+time_range_dict = {'day': 60 * 60 * 24,
36
+                   'week': 60 * 60 * 24 * 7,
37
+                   'month': 60 * 60 * 24 * 7 * 4}
32 38
 
33 39
 
34 40
 def build_flickr_url(user_id, photo_id):
35 41
     return photo_url.format(userid=user_id, photoid=photo_id)
36 42
 
37 43
 
38
-def request(query, params):
39
-    params['url'] = search_url.format(query=urlencode({'text': query}),
40
-                                      page=params['pageno'])
44
+def _get_time_range_url(time_range):
45
+    if time_range in time_range_dict:
46
+        return time_range_url.format(start=time(), end=str(int(time()) - time_range_dict[time_range]))
47
+    return ''
48
+
41 49
 
50
+def request(query, params):
51
+    params['url'] = (search_url.format(query=urlencode({'text': query}), page=params['pageno'])
52
+                     + _get_time_range_url(params['time_range']))
42 53
     return params
43 54
 
44 55
 

+ 7
- 0
searx/engines/youtube_noapi.py Zobrazit soubor

@@ -17,10 +17,15 @@ from searx.utils import list_get
17 17
 categories = ['videos', 'music']
18 18
 paging = True
19 19
 language_support = False
20
+time_range_support = True
20 21
 
21 22
 # search-url
22 23
 base_url = 'https://www.youtube.com/results'
23 24
 search_url = base_url + '?search_query={query}&page={page}'
25
+time_range_url = '&sp=EgII{time_range}%253D%253D'
26
+time_range_dict = {'day': 'Ag',
27
+                   'week': 'Aw',
28
+                   'month': 'BA'}
24 29
 
25 30
 embedded_url = '<iframe width="540" height="304" ' +\
26 31
     'data-src="//www.youtube-nocookie.com/embed/{videoid}" ' +\
@@ -47,6 +52,8 @@ def extract_text_from_dom(result, xpath):
47 52
 def request(query, params):
48 53
     params['url'] = search_url.format(query=quote_plus(query),
49 54
                                       page=params['pageno'])
55
+    if params['time_range'] in time_range_dict:
56
+        params['url'] += time_range_url.format(time_range=time_range_dict[params['time_range']])
50 57
 
51 58
     return params
52 59
 

+ 2
- 0
searx/templates/oscar/preferences.html Zobrazit soubor

@@ -158,6 +158,7 @@
158 158
 				    <th>{{ _("Engine name") }}</th>
159 159
 				    <th>{{ _("Shortcut") }}</th>
160 160
 				    <th>{{ _("SafeSearch") }}</th>
161
+				    <th>{{ _("Time range") }}</th>
161 162
 				    <th>{{ _("Avg. time") }}</th>
162 163
 				    <th>{{ _("Max time") }}</th>
163 164
                                     {% else %}
@@ -179,6 +180,7 @@
179 180
                                     <th>{{ search_engine.name }}</th>
180 181
 				    <td>{{ shortcuts[search_engine.name] }}</td>
181 182
 				    <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td>
183
+				    <td><input type="checkbox" {{ "checked" if search_engine.time_range_support==True else ""}} readonly="readonly" disabled="disabled"></td>
182 184
 				    <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>
183 185
 				    <td class="{{ 'danger' if stats[search_engine.name]['warn_timeout'] else '' }}">{{ search_engine.timeout }}</td>
184 186
                                     {% else %}

+ 1
- 0
tests/unit/engines/test_bing_images.py Zobrazit soubor

@@ -13,6 +13,7 @@ class TestBingImagesEngine(SearxTestCase):
13 13
         dicto['pageno'] = 1
14 14
         dicto['language'] = 'fr_FR'
15 15
         dicto['safesearch'] = 1
16
+        dicto['time_range'] = ''
16 17
         params = bing_images.request(query, dicto)
17 18
         self.assertTrue('url' in params)
18 19
         self.assertTrue(query in params['url'])

+ 1
- 0
tests/unit/engines/test_bing_news.py Zobrazit soubor

@@ -12,6 +12,7 @@ class TestBingNewsEngine(SearxTestCase):
12 12
         dicto = defaultdict(dict)
13 13
         dicto['pageno'] = 1
14 14
         dicto['language'] = 'fr_FR'
15
+        dicto['time_range'] = ''
15 16
         params = bing_news.request(query, dicto)
16 17
         self.assertIn('url', params)
17 18
         self.assertIn(query, params['url'])

+ 1
- 0
tests/unit/engines/test_flickr_noapi.py Zobrazit soubor

@@ -15,6 +15,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
15 15
         query = 'test_query'
16 16
         dicto = defaultdict(dict)
17 17
         dicto['pageno'] = 1
18
+        dicto['time_range'] = ''
18 19
         params = flickr_noapi.request(query, dicto)
19 20
         self.assertIn('url', params)
20 21
         self.assertIn(query, params['url'])

+ 1
- 0
tests/unit/engines/test_youtube_noapi.py Zobrazit soubor

@@ -11,6 +11,7 @@ class TestYoutubeNoAPIEngine(SearxTestCase):
11 11
         query = 'test_query'
12 12
         dicto = defaultdict(dict)
13 13
         dicto['pageno'] = 0
14
+        dicto['time_range'] = ''
14 15
         params = youtube_noapi.request(query, dicto)
15 16
         self.assertIn('url', params)
16 17
         self.assertIn(query, params['url'])