Pārlūkot izejas kodu

add time range search with yahoo

Noemi Vanyi 8 gadus atpakaļ
vecāks
revīzija
93c0c49e9a
4 mainītis faili ar 35 papildinājumiem un 11 dzēšanām
  1. 2
    1
      searx/engines/__init__.py
  2. 25
    8
      searx/engines/yahoo.py
  3. 7
    2
      searx/search.py
  4. 1
    0
      searx/webapp.py

+ 2
- 1
searx/engines/__init__.py Parādīt failu

42
                        'shortcut': '-',
42
                        'shortcut': '-',
43
                        'disabled': False,
43
                        'disabled': False,
44
                        'suspend_end_time': 0,
44
                        'suspend_end_time': 0,
45
-                       'continuous_errors': 0}
45
+                       'continuous_errors': 0,
46
+                       'time_range_support': False}
46
 
47
 
47
 
48
 
48
 def load_module(filename):
49
 def load_module(filename):

+ 25
- 8
searx/engines/yahoo.py Parādīt failu

20
 categories = ['general']
20
 categories = ['general']
21
 paging = True
21
 paging = True
22
 language_support = True
22
 language_support = True
23
+time_range_support = True
23
 
24
 
24
 # search-url
25
 # search-url
25
 base_url = 'https://search.yahoo.com/'
26
 base_url = 'https://search.yahoo.com/'
26
 search_url = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}'
27
 search_url = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}'
28
+search_url_with_time = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}&age={age}&btf={btf}&fr2=time'
27
 
29
 
28
 # specific xpath variables
30
 # specific xpath variables
29
 results_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' Sr ')]"
31
 results_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' Sr ')]"
32
 content_xpath = './/div[@class="compText aAbs"]'
34
 content_xpath = './/div[@class="compText aAbs"]'
33
 suggestion_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' AlsoTry ')]//a"
35
 suggestion_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' AlsoTry ')]//a"
34
 
36
 
37
+time_range_dict = {'day': ['1d', 'd'],
38
+                   'week': ['1w', 'w'],
39
+                   'month': ['1m', 'm']}
35
 
40
 
36
 # remove yahoo-specific tracking-url
41
 # remove yahoo-specific tracking-url
37
 def parse_url(url_string):
42
 def parse_url(url_string):
51
         return unquote(url_string[start:end])
56
         return unquote(url_string[start:end])
52
 
57
 
53
 
58
 
59
+def _get_url(query, offset, language, time_range):
60
+    if time_range:
61
+        return base_url + search_url_with_time.format(offset=offset,
62
+                                                      query=urlencode({'p': query}),
63
+                                                      lang=language,
64
+                                                      age=time_range_dict[time_range][0],
65
+                                                      btf=time_range_dict[time_range][1])
66
+    return base_url + search_url.format(offset=offset,
67
+                                        query=urlencode({'p': query}),
68
+                                        lang=language)
69
+
70
+
71
+def _get_language(params):
72
+    if params['language'] == 'all':
73
+        return 'en'
74
+    return params['language'].split('_')[0]
75
+
76
+
54
 # do search-request
77
 # do search-request
55
 def request(query, params):
78
 def request(query, params):
56
     offset = (params['pageno'] - 1) * 10 + 1
79
     offset = (params['pageno'] - 1) * 10 + 1
80
+    language = _get_language(params)
57
 
81
 
58
-    if params['language'] == 'all':
59
-        language = 'en'
60
-    else:
61
-        language = params['language'].split('_')[0]
62
-
63
-    params['url'] = base_url + search_url.format(offset=offset,
64
-                                                 query=urlencode({'p': query}),
65
-                                                 lang=language)
82
+    params['url'] = _get_url(query, offset, language, params['time_range'])
66
 
83
 
67
     # TODO required?
84
     # TODO required?
68
     params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\
85
     params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\

+ 7
- 2
searx/search.py Parādīt failu

138
         self.paging = False
138
         self.paging = False
139
         self.pageno = 1
139
         self.pageno = 1
140
         self.lang = 'all'
140
         self.lang = 'all'
141
+        self.time_range = None
141
 
142
 
142
         # set blocked engines
143
         # set blocked engines
143
         self.disabled_engines = request.preferences.engines.get_disabled()
144
         self.disabled_engines = request.preferences.engines.get_disabled()
178
         if len(query_obj.languages):
179
         if len(query_obj.languages):
179
             self.lang = query_obj.languages[-1]
180
             self.lang = query_obj.languages[-1]
180
 
181
 
181
-        self.engines = query_obj.engines
182
+        self.time_range = self.request_data.get('time_range')
182
 
183
 
183
-        self.categories = []
184
+        self.engines = query_obj.engines
184
 
185
 
185
         # if engines are calculated from query,
186
         # if engines are calculated from query,
186
         # set categories by using that informations
187
         # set categories by using that informations
279
             if self.lang != 'all' and not engine.language_support:
280
             if self.lang != 'all' and not engine.language_support:
280
                 continue
281
                 continue
281
 
282
 
283
+            if self.time_range and not engine.time_range_support:
284
+                continue
285
+
282
             # set default request parameters
286
             # set default request parameters
283
             request_params = default_request_params()
287
             request_params = default_request_params()
284
             request_params['headers']['User-Agent'] = user_agent
288
             request_params['headers']['User-Agent'] = user_agent
293
 
297
 
294
             # 0 = None, 1 = Moderate, 2 = Strict
298
             # 0 = None, 1 = Moderate, 2 = Strict
295
             request_params['safesearch'] = request.preferences.get_value('safesearch')
299
             request_params['safesearch'] = request.preferences.get_value('safesearch')
300
+            request_params['time_range'] = self.time_range
296
 
301
 
297
             # update request parameters dependent on
302
             # update request parameters dependent on
298
             # search-engine (contained in engines folder)
303
             # search-engine (contained in engines folder)

+ 1
- 0
searx/webapp.py Parādīt failu

459
         paging=search.paging,
459
         paging=search.paging,
460
         number_of_results=format_decimal(number_of_results),
460
         number_of_results=format_decimal(number_of_results),
461
         pageno=search.pageno,
461
         pageno=search.pageno,
462
+        time_range=search.time_range,
462
         base_url=get_base_url(),
463
         base_url=get_base_url(),
463
         suggestions=search.result_container.suggestions,
464
         suggestions=search.result_container.suggestions,
464
         answers=search.result_container.answers,
465
         answers=search.result_container.answers,