Browse Source

Deviant Art's unit test

Cqoicebordel 10 years ago
parent
commit
d495704551

+ 4
- 3
searx/engines/deviantart.py View File

@@ -14,6 +14,7 @@ from urllib import urlencode
14 14
 from urlparse import urljoin
15 15
 from lxml import html
16 16
 import re
17
+from searx.engines.xpath import extract_text
17 18
 
18 19
 # engine dependent config
19 20
 categories = ['images']
@@ -50,9 +51,9 @@ def response(resp):
50 51
     for result in dom.xpath('//div[contains(@class, "tt-a tt-fh")]'):
51 52
         link = result.xpath('.//a[contains(@class, "thumb")]')[0]
52 53
         url = urljoin(base_url, link.attrib.get('href'))
53
-        title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]')  # noqa
54
-        title = ''.join(title_links[0].xpath('.//text()'))
55
-        thumbnail_src = link.xpath('.//img')[0].attrib['src']
54
+        title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]')
55
+        title = extract_text(title_links[0])
56
+        thumbnail_src = link.xpath('.//img')[0].attrib.get('src')
56 57
         img_src = regex.sub('/', thumbnail_src)
57 58
 
58 59
         # append result

+ 118
- 0
searx/tests/engines/test_deviantart.py View File

@@ -0,0 +1,118 @@
1
+from collections import defaultdict
2
+import mock
3
+from searx.engines import deviantart
4
+from searx.testing import SearxTestCase
5
+
6
+
7
+class TestDeviantartEngine(SearxTestCase):
8
+
9
+    def test_request(self):
10
+        query = 'test_query'
11
+        dicto = defaultdict(dict)
12
+        dicto['pageno'] = 0
13
+        params = deviantart.request(query, dicto)
14
+        self.assertTrue('url' in params)
15
+        self.assertTrue(query in params['url'])
16
+        self.assertTrue('deviantart.com' in params['url'])
17
+
18
+    def test_response(self):
19
+        self.assertRaises(AttributeError, deviantart.response, None)
20
+        self.assertRaises(AttributeError, deviantart.response, [])
21
+        self.assertRaises(AttributeError, deviantart.response, '')
22
+        self.assertRaises(AttributeError, deviantart.response, '[]')
23
+
24
+        response = mock.Mock(text='<html></html>')
25
+        self.assertEqual(deviantart.response(response), [])
26
+
27
+        response = mock.Mock(status_code=302)
28
+        self.assertEqual(deviantart.response(response), [])
29
+
30
+        html = """
31
+        <div class="tt-a tt-fh tt-boxed" collect_rid="1:149167425"
32
+            usericon="http://a.deviantart.net/avatars/t/e/test-0.gif" userid="233301"
33
+            username="test-0" symbol="~" category="digitalart/animation">
34
+            <span class="tt-w" style="width: auto; max-width: 277px;">
35
+                <span class="tt-fh-tc" style="width: 202px;">
36
+                    <span class="tt-bb" style="width: 202px;">
37
+                    </span>
38
+                    <span class="shadow">
39
+                        <a class="thumb" href="http://url.of.result/2nd.part.of.url"
40
+                            title="Behoimi BE Animation Test by test-0, Jan 4,
41
+                            2010 in Digital Art &gt; Animation"> <i></i>
42
+                            <img width="200" height="200" alt="Test"
43
+                                src="http://url.of.thumbnail" data-src="http://th08.deviantart.net/test.jpg">
44
+                        </a>
45
+                    </span>
46
+                    <!-- ^TTT -->
47
+                </span>
48
+                <span class="details">
49
+                    <a href="http://test-0.deviantart.com/art/Test" class="t"
50
+                        title="Behoimi BE Animation Test by test-0, Jan 4, 2010">
51
+                        <span class="tt-fh-oe">Title of image</span> </a>
52
+                    <small>
53
+                    <span class="category">
54
+                        <span class="age">
55
+                            5 years ago
56
+                        </span>
57
+                        in <a title="Behoimi BE Animation Test by test-0, Jan 4, 2010"
58
+                            href="http://www.deviantart.com/browse/all/digitalart/animation/">Animation</a>
59
+                    </span>
60
+                    <div class="commentcount">
61
+                        <a href="http://test-0.deviantart.com/art/Test#comments">
62
+                        <span class="iconcommentsstats"></span>9 Comments</a>
63
+                    </div>
64
+                    <a class="mlt-link" href="http://www.deviantart.com/morelikethis/149167425">
65
+                    <span class="mlt-icon"></span> <span class="mlt-text">More Like This</span> </a>
66
+                </span>
67
+                </small> <!-- TTT$ -->
68
+            </span>
69
+        </div>
70
+        """
71
+        response = mock.Mock(text=html)
72
+        results = deviantart.response(response)
73
+        self.assertEqual(type(results), list)
74
+        self.assertEqual(len(results), 1)
75
+        self.assertEqual(results[0]['title'], 'Title of image')
76
+        self.assertEqual(results[0]['url'], 'http://url.of.result/2nd.part.of.url')
77
+        self.assertNotIn('content', results[0])
78
+        self.assertEqual(results[0]['thumbnail_src'], 'http://url.of.thumbnail')
79
+
80
+        html = """
81
+        <span class="tt-fh-tc" style="width: 202px;">
82
+            <span class="tt-bb" style="width: 202px;">
83
+            </span>
84
+            <span class="shadow">
85
+                <a class="thumb" href="http://url.of.result/2nd.part.of.url"
86
+                    title="Behoimi BE Animation Test by test-0, Jan 4,
87
+                    2010 in Digital Art &gt; Animation"> <i></i>
88
+                    <img width="200" height="200" alt="Test"
89
+                        src="http://url.of.thumbnail" data-src="http://th08.deviantart.net/test.jpg">
90
+                </a>
91
+            </span>
92
+            <!-- ^TTT -->
93
+        </span>
94
+        <span class="details">
95
+            <a href="http://test-0.deviantart.com/art/Test" class="t"
96
+                title="Behoimi BE Animation Test by test-0, Jan 4, 2010">
97
+                <span class="tt-fh-oe">Title of image</span> </a>
98
+            <small>
99
+            <span class="category">
100
+                <span class="age">
101
+                    5 years ago
102
+                </span>
103
+                in <a title="Behoimi BE Animation Test by test-0, Jan 4, 2010"
104
+                    href="http://www.deviantart.com/browse/all/digitalart/animation/">Animation</a>
105
+            </span>
106
+            <div class="commentcount">
107
+                <a href="http://test-0.deviantart.com/art/Test#comments">
108
+                <span class="iconcommentsstats"></span>9 Comments</a>
109
+            </div>
110
+            <a class="mlt-link" href="http://www.deviantart.com/morelikethis/149167425">
111
+            <span class="mlt-icon"></span> <span class="mlt-text">More Like This</span> </a>
112
+        </span>
113
+        </small> <!-- TTT$ -->
114
+        """
115
+        response = mock.Mock(text=html)
116
+        results = deviantart.response(response)
117
+        self.assertEqual(type(results), list)
118
+        self.assertEqual(len(results), 0)

+ 45
- 1
searx/tests/engines/test_digg.py View File

@@ -32,9 +32,53 @@ class TestDiggEngine(SearxTestCase):
32 32
         "status": "ok",
33 33
         "num": 10,
34 34
         "next_position": 20,
35
-        "html": "<article itemscope itemtype=\\"http://schema.org/Article\\" class=\\"story-container digg-story-el hentry entry story-1sRANah col-1\\" data-content-id=\\"1sRANah\\" data-contenturl=\\"http://url.of.link\\" data-position=\\"0\\" data-diggs=\\"24\\" data-tweets=\\"69\\" data-digg-score=\\"1190\\"> <div class=\\"story-image story-image-thumb\\"> <a data-position=\\"0\\" data-content-id=\\"1sRANah\\" class=\\"story-link\\" href=\\"http://www.thedailybeast.com/\\" target=\\"_blank\\"><img class=\\"story-image-img\\" src=\\"http://url.of.image.jpeg\\" width=\\"312\\" height=\\"170\\" alt=\\"\\" /> </a> </div> <div class=\\"story-content\\"><header class=\\"story-header\\"> <div itemprop=\\"alternativeHeadline\\" class=\\"story-kicker\\" >Kicker</div> <h2 itemprop=\\"headline\\" class=\\"story-title entry-title\\"><a class=\\"story-title-link story-link\\" rel=\\"bookmark\\" itemprop=\\"url\\" href=\\"http://www.thedailybeast.com/\\" target=\\"_blank\\">Title of article</h2> <div class=\\"story-meta\\"><div class=\\"story-score \\"> <div class=\\"story-score-diggscore diggscore-1sRANah\\">1190</div> <div class=\\"story-score-details\\"> <div class=\\"arrow\\"></div><ul class=\\"story-score-details-list\\"> <li class=\\"story-score-detail story-score-diggs\\"><span class=\\"label\\">Diggs:</span> <span class=\\"count diggs-1sRANah\\">24</span></li> <li class=\\"story-score-detail story-score-twitter\\"><span class=\\"label\\">Tweets:</span> <span class=\\"count tweets-1sRANah\\">69</span></li> <li class=\\"story-score-detail story-score-facebook\\"><span class=\\"label\\">Facebook Shares:</span> <span class=\\"count fb_shares-1sRANah\\">1097</span></li> </ul> </div> </div> <span class=\\"story-meta-item story-source\\"> <a itemprop=\\"publisher copyrightHolder sourceOrganization provider\\" class=\\"story-meta-item-link story-source-link\\" href=\\"/source/thedailybeast.com\\">The Daily Beast </a> </span> <span class=\\"story-meta-item story-tag first-tag\\"> <a itemprop=\\"keywords\\" rel=\\"tag\\" class=\\"story-meta-item-link story-tag-link\\" href=\\"/tag/news\\">News</a> </span> <abbr class=\\"published story-meta-item story-timestamp\\" title=\\"2014-10-18 14:53:45\\"> <time datetime=\\"2014-10-18 14:53:45\\">18 Oct 2014</time> </abbr> </div> </header> </div> <ul class=\\"story-actions\\"> <li class=\\"story-action story-action-digg btn-story-action-container\\"><a class=\\"target digg-1sRANah\\" href=\\"#\\">Digg</a></li> <li class=\\"story-action story-action-save btn-story-action-container\\"><a class=\\"target save-1sRANah\\" href=\\"#\\">Save</a></li> <li class=\\"story-action story-action-share\\"><a class=\\"target share-facebook\\" href=\\"https://www.facebook.com/\\">Facebook</a></li> <li class=\\"story-action story-action-share\\"><a class=\\"target share-twitter\\" href=\\"https://twitter.com/\\">Twitter</a></li> </ul> </article>"
35
+        "html": "<article itemscope itemtype=\\"http://schema.org/Article\\"
36
+        class=\\"story-container digg-story-el hentry entry story-1sRANah col-1\\"
37
+        data-content-id=\\"1sRANah\\" data-contenturl=\\"http://url.of.link\\"
38
+        data-position=\\"0\\" data-diggs=\\"24\\" data-tweets=\\"69\\"
39
+        data-digg-score=\\"1190\\"> <div class=\\"story-image story-image-thumb\\">
40
+        <a data-position=\\"0\\" data-content-id=\\"1sRANah\\"
41
+        class=\\"story-link\\" href=\\"http://www.thedailybeast.com/\\"
42
+        target=\\"_blank\\"><img class=\\"story-image-img\\"
43
+        src=\\"http://url.of.image.jpeg\\" width=\\"312\\" height=\\"170\\"
44
+        alt=\\"\\" /> </a> </div> <div class=\\"story-content\\"><header
45
+        class=\\"story-header\\"> <div itemprop=\\"alternativeHeadline\\"
46
+        class=\\"story-kicker\\" >Kicker</div> <h2 itemprop=\\"headline\\"
47
+        class=\\"story-title entry-title\\"><a class=\\"story-title-link story-link\\"
48
+        rel=\\"bookmark\\" itemprop=\\"url\\" href=\\"http://www.thedailybeast.com/\\"
49
+        target=\\"_blank\\">Title of article</h2> <div class=\\"story-meta\\">
50
+        <div class=\\"story-score \\">
51
+        <div class=\\"story-score-diggscore diggscore-1sRANah\\">1190</div>
52
+        <div class=\\"story-score-details\\"> <div class=\\"arrow\\"></div>
53
+        <ul class=\\"story-score-details-list\\"> <li
54
+        class=\\"story-score-detail story-score-diggs\\"><span
55
+        class=\\"label\\">Diggs:</span> <span class=\\"count diggs-1sRANah\\">24</span>
56
+        </li> <li class=\\"story-score-detail story-score-twitter\\"><span
57
+        class=\\"label\\">Tweets:</span> <span class=\\"count tweets-1sRANah\\">69</span>
58
+        </li> <li class=\\"story-score-detail story-score-facebook\\"><span
59
+        class=\\"label\\">Facebook Shares:</span> <span
60
+        class=\\"count fb_shares-1sRANah\\">1097</span></li> </ul> </div> </div>
61
+        <span class=\\"story-meta-item story-source\\"> <a
62
+        itemprop=\\"publisher copyrightHolder sourceOrganization provider\\"
63
+        class=\\"story-meta-item-link story-source-link\\"
64
+        href=\\"/source/thedailybeast.com\\">The Daily Beast </a> </span>
65
+        <span class=\\"story-meta-item story-tag first-tag\\"> <a
66
+        itemprop=\\"keywords\\" rel=\\"tag\\"
67
+        class=\\"story-meta-item-link story-tag-link\\" href=\\"/tag/news\\">News</a>
68
+        </span> <abbr class=\\"published story-meta-item story-timestamp\\"
69
+        title=\\"2014-10-18 14:53:45\\"> <time datetime=\\"2014-10-18 14:53:45\\">18 Oct 2014</time>
70
+        </abbr> </div> </header> </div> <ul class=\\"story-actions\\"> <li
71
+        class=\\"story-action story-action-digg btn-story-action-container\\">
72
+        <a class=\\"target digg-1sRANah\\" href=\\"#\\">Digg</a></li> <li
73
+        class=\\"story-action story-action-save btn-story-action-container\\">
74
+        <a class=\\"target save-1sRANah\\" href=\\"#\\">Save</a></li> <li
75
+        class=\\"story-action story-action-share\\"><a
76
+        class=\\"target share-facebook\\" href=\\"https://www.facebook.com/\\">Facebook</a></li>
77
+        <li class=\\"story-action story-action-share\\"><a class=\\"target share-twitter\\"
78
+        href=\\"https://twitter.com/\\">Twitter</a></li> </ul> </article>"
36 79
         }
37 80
         """
81
+        json = json.replace('\r\n', '').replace('\n', '').replace('\r', '')
38 82
         response = mock.Mock(text=json)
39 83
         results = digg.response(response)
40 84
         self.assertEqual(type(results), list)

+ 1
- 0
searx/tests/test_engines.py View File

@@ -1,6 +1,7 @@
1 1
 from searx.tests.engines.test_bing import *  # noqa
2 2
 from searx.tests.engines.test_dailymotion import *  # noqa
3 3
 from searx.tests.engines.test_deezer import *  # noqa
4
+from searx.tests.engines.test_deviantart import *  # noqa
4 5
 from searx.tests.engines.test_digg import *  # noqa
5 6
 from searx.tests.engines.test_dummy import *  # noqa
6 7
 from searx.tests.engines.test_flickr import *  # noqa