Browse Source

[fix] broken google images parsing

Adam Tauber 9 years ago
parent
commit
9331fc28a8
2 changed files with 6 additions and 8 deletions
  1. 3
    5
      searx/engines/google_images.py
  2. 3
    3
      tests/unit/engines/test_google_images.py

+ 3
- 5
searx/engines/google_images.py View File

49
 
49
 
50
     # parse results
50
     # parse results
51
     for result in dom.xpath('//div[@data-ved]'):
51
     for result in dom.xpath('//div[@data-ved]'):
52
-        data_url = result.xpath('./a/@href')[0]
53
-        data_query = {k: v[0] for k, v in parse_qs(data_url.split('?', 1)[1]).iteritems()}
54
 
52
 
55
         metadata = loads(result.xpath('./div[@class="rg_meta"]/text()')[0])
53
         metadata = loads(result.xpath('./div[@class="rg_meta"]/text()')[0])
56
 
54
 
60
         thumbnail_src = thumbnail_src.replace("http://", "https://")
58
         thumbnail_src = thumbnail_src.replace("http://", "https://")
61
 
59
 
62
         # append result
60
         # append result
63
-        results.append({'url': data_query['imgrefurl'],
61
+        results.append({'url': metadata['ru'],
64
                         'title': metadata['pt'],
62
                         'title': metadata['pt'],
65
                         'content': metadata['s'],
63
                         'content': metadata['s'],
66
-                        'thumbnail_src': metadata['tu'],
67
-                        'img_src': data_query['imgurl'],
64
+                        'thumbnail_src': thumbnail_src,
65
+                        'img_src': metadata['ou'],
68
                         'template': 'images.html'})
66
                         'template': 'images.html'})
69
 
67
 
70
     # return results
68
     # return results

+ 3
- 3
tests/unit/engines/test_google_images.py View File

41
         </div>
41
         </div>
42
       </a>
42
       </a>
43
       <div class="rg_meta">
43
       <div class="rg_meta">
44
-        {"id":"bQWQ9wz9loJmjM:","isu":"clker.com","ity":"png","md":"/search?tbs\u003dsbi:AMhZZit7u1mHyop9pQisu-5idR-8W_1Itvwc3afChmsjQYPx_1yYMzBvUZgtkcGoojqekKZ-6n_1rjX9ySH0OWA_1eO5OijFY6BBDw_1GApr6xxb1bXJcBcj-DiguMoXWW7cZSG7MRQbwnI5SoDZNXcv_1xGszy886I7NVb_1oRKSliTHtzqbXAxhvYreM","msu":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAQSEgltBZD3DP2WgiG-U42R4G0RFw","oh":598,"os":"13KB","ow":504,"pt":"South Arrow Clip Art at Clker.com - vector clip art online ...","rid":"vlONkeBtERfDuM","s":"Download this image as:","sc":1,"si":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAESEgltBZD3DP2WgiG-U42R4G0RFw","th":245,"tu":"https://thumbnail.url/","tw":206}
44
+        {"id":"bQWQ9wz9loJmjM:","isu":"clker.com","ity":"png","md":"/search?tbs\u003dsbi:AMhZZit7u1mHyop9pQisu-5idR-8W_1Itvwc3afChmsjQYPx_1yYMzBvUZgtkcGoojqekKZ-6n_1rjX9ySH0OWA_1eO5OijFY6BBDw_1GApr6xxb1bXJcBcj-DiguMoXWW7cZSG7MRQbwnI5SoDZNXcv_1xGszy886I7NVb_1oRKSliTHtzqbXAxhvYreM","msu":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAQSEgltBZD3DP2WgiG-U42R4G0RFw","oh":598,"os":"13KB","ow":504,"pt":"South Arrow Clip Art at Clker.com - vector clip art online ...","rid":"vlONkeBtERfDuM","s":"Download this image as:","sc":1,"si":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAESEgltBZD3DP2WgiG-U42R4G0RFw","th":245,"tu":"https://thumbnail.url/","tw":206,"ru":"a","ou":"b"}
45
       </div>
45
       </div>
46
     </div><!--n--><!--m-->
46
     </div><!--n--><!--m-->
47
   </div>
47
   </div>
52
         self.assertEqual(type(results), list)
52
         self.assertEqual(type(results), list)
53
         self.assertEqual(len(results), 1)
53
         self.assertEqual(len(results), 1)
54
         self.assertEqual(results[0]['title'], u'South Arrow Clip Art at Clker.com - vector clip art online ...')
54
         self.assertEqual(results[0]['title'], u'South Arrow Clip Art at Clker.com - vector clip art online ...')
55
-        self.assertEqual(results[0]['url'], 'http://www.clker.com/clipart-south-arrow.html')
55
+        self.assertEqual(results[0]['url'], 'a')
56
         self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url/')
56
         self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url/')
57
-        self.assertEqual(results[0]['img_src'], 'http://www.clker.com/cliparts/H/X/l/b/0/0/south-arrow-hi.png')
57
+        self.assertEqual(results[0]['img_src'], 'b')
58
         self.assertEqual(results[0]['content'], 'Download this image as:')
58
         self.assertEqual(results[0]['content'], 'Download this image as:')