Bläddra i källkod

simplify datetime extraction

Thomas Pointhuber 11 år sedan
förälder
incheckning
337bd6d907
5 ändrade filer med 15 tillägg och 13 borttagningar
  1. 1
    0
      requirements.txt
  2. 4
    10
      searx/engines/google_news.py
  3. 2
    3
      searx/engines/yahoo_news.py
  4. 7
    0
      searx/engines/youtube.py
  5. 1
    0
      setup.py

+ 1
- 0
requirements.txt Visa fil

3
 grequests
3
 grequests
4
 lxml
4
 lxml
5
 pyyaml
5
 pyyaml
6
+python-dateutil

+ 4
- 10
searx/engines/google_news.py Visa fil

2
 
2
 
3
 from urllib import urlencode
3
 from urllib import urlencode
4
 from json import loads
4
 from json import loads
5
+from dateutil import parser
5
 from datetime import datetime
6
 from datetime import datetime
6
 
7
 
7
 categories = ['news']
8
 categories = ['news']
32
         return []
33
         return []
33
 
34
 
34
     for result in search_res['responseData']['results']:
35
     for result in search_res['responseData']['results']:
35
-# S.149 (159), library.pdf
36
-# datetime.strptime("Mon, 10 Mar 2014 16:26:15 -0700",
37
-#                   "%a, %d %b %Y %H:%M:%S %z")
38
-#        publishedDate = parse(result['publishedDate'])
39
-        publishedDate = datetime.strptime(
40
-            str.join(' ', result['publishedDate'].split(None)[0:5]),
41
-            "%a, %d %b %Y %H:%M:%S")
42
-        #utc_offset = timedelta(result['publishedDate'].split(None)[5])
43
-        # local = utc + offset
44
-        #publishedDate = publishedDate + utc_offset
36
+
37
+# Mon, 10 Mar 2014 16:26:15 -0700
38
+        publishedDate = parser.parse(result['publishedDate'])
45
 
39
 
46
         results.append({'url': result['unescapedUrl'],
40
         results.append({'url': result['unescapedUrl'],
47
                         'title': result['titleNoFormatting'],
41
                         'title': result['titleNoFormatting'],

+ 2
- 3
searx/engines/yahoo_news.py Visa fil

6
 from searx.engines.yahoo import parse_url
6
 from searx.engines.yahoo import parse_url
7
 from datetime import datetime, timedelta
7
 from datetime import datetime, timedelta
8
 import re
8
 import re
9
+from dateutil import parser
9
 
10
 
10
 categories = ['news']
11
 categories = ['news']
11
 search_url = 'http://news.search.yahoo.com/search?{query}&b={offset}'
12
 search_url = 'http://news.search.yahoo.com/search?{query}&b={offset}'
52
                     - timedelta(hours=int(timeNumbers[0]))\
53
                     - timedelta(hours=int(timeNumbers[0]))\
53
                     - timedelta(minutes=int(timeNumbers[1]))
54
                     - timedelta(minutes=int(timeNumbers[1]))
54
             else:
55
             else:
55
-                # TODO year in string possible?
56
-                publishedDate = datetime.strptime(publishedDate,
57
-                                                  "%b %d %H:%M%p")
56
+                publishedDate =parser.parse(publishedDate)
58
 
57
 
59
         if publishedDate.year == 1900:
58
         if publishedDate.year == 1900:
60
             publishedDate = publishedDate.replace(year=datetime.now().year)
59
             publishedDate = publishedDate.replace(year=datetime.now().year)

+ 7
- 0
searx/engines/youtube.py Visa fil

1
 from json import loads
1
 from json import loads
2
 from urllib import urlencode
2
 from urllib import urlencode
3
+from dateutil import parser
4
+from datetime import datetime
3
 
5
 
4
 categories = ['videos']
6
 categories = ['videos']
5
 
7
 
35
         content = ''
37
         content = ''
36
         thumbnail = ''
38
         thumbnail = ''
37
 
39
 
40
+#"2013-12-31T15:22:51.000Z"
41
+        pubdate = result['published']['$t']
42
+        publishedDate = parser.parse(pubdate)
43
+
38
         if result['media$group']['media$thumbnail']:
44
         if result['media$group']['media$thumbnail']:
39
             thumbnail = result['media$group']['media$thumbnail'][0]['url']
45
             thumbnail = result['media$group']['media$thumbnail'][0]['url']
40
             content += '<a href="{0}" title="{0}" ><img src="{1}" /></a>'.format(url, thumbnail)  # noqa
46
             content += '<a href="{0}" title="{0}" ><img src="{1}" /></a>'.format(url, thumbnail)  # noqa
48
                         'title': title,
54
                         'title': title,
49
                         'content': content,
55
                         'content': content,
50
                         'template': 'videos.html',
56
                         'template': 'videos.html',
57
+                        'publishedDate': publishedDate,
51
                         'thumbnail': thumbnail})
58
                         'thumbnail': thumbnail})
52
 
59
 
53
     return results
60
     return results

+ 1
- 0
setup.py Visa fil

35
         'lxml',
35
         'lxml',
36
         'pyyaml',
36
         'pyyaml',
37
         'setuptools',
37
         'setuptools',
38
+        'python-dateutil',
38
     ],
39
     ],
39
     extras_require={
40
     extras_require={
40
         'test': [
41
         'test': [