Browse Source

Kickass' unit test

Cqoicebordel 10 years ago
parent
commit
8ea749d6ec
3 changed files with 403 additions and 4 deletions
  1. 4
    4
      searx/engines/kickass.py
  2. 398
    0
      searx/tests/engines/test_kickass.py
  3. 1
    0
      searx/tests/test_engines.py

+ 4
- 4
searx/engines/kickass.py View File

@@ -13,6 +13,7 @@ from cgi import escape
13 13
 from urllib import quote
14 14
 from lxml import html
15 15
 from operator import itemgetter
16
+from searx.engines.xpath import extract_text
16 17
 
17 18
 # engine dependent config
18 19
 categories = ['videos', 'music', 'files']
@@ -56,9 +57,8 @@ def response(resp):
56 57
     for result in search_res[1:]:
57 58
         link = result.xpath('.//a[@class="cellMainLink"]')[0]
58 59
         href = urljoin(url, link.attrib['href'])
59
-        title = ' '.join(link.xpath('.//text()'))
60
-        content = escape(html.tostring(result.xpath(content_xpath)[0],
61
-                                       method="text"))
60
+        title = extract_text(link)
61
+        content = escape(extract_text(result.xpath(content_xpath)))
62 62
         seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]
63 63
         leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]
64 64
         filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0]
@@ -88,7 +88,7 @@ def response(resp):
88 88
                 filesize = int(filesize * 1024 * 1024 * 1024)
89 89
             elif filesize_multiplier == 'MB':
90 90
                 filesize = int(filesize * 1024 * 1024)
91
-            elif filesize_multiplier == 'kb':
91
+            elif filesize_multiplier == 'KB':
92 92
                 filesize = int(filesize * 1024)
93 93
         except:
94 94
             filesize = None

+ 398
- 0
searx/tests/engines/test_kickass.py View File

@@ -0,0 +1,398 @@
1
+# -*- coding: utf-8 -*-
2
+from collections import defaultdict
3
+import mock
4
+from searx.engines import kickass
5
+from searx.testing import SearxTestCase
6
+
7
+
8
+class TestKickassEngine(SearxTestCase):
9
+
10
+    def test_request(self):
11
+        query = 'test_query'
12
+        dicto = defaultdict(dict)
13
+        dicto['pageno'] = 1
14
+        params = kickass.request(query, dicto)
15
+        self.assertIn('url', params)
16
+        self.assertIn(query, params['url'])
17
+        self.assertIn('kickass.so', params['url'])
18
+        self.assertIn('verify', params)
19
+        self.assertFalse(params['verify'])
20
+
21
+    def test_response(self):
22
+        self.assertRaises(AttributeError, kickass.response, None)
23
+        self.assertRaises(AttributeError, kickass.response, [])
24
+        self.assertRaises(AttributeError, kickass.response, '')
25
+        self.assertRaises(AttributeError, kickass.response, '[]')
26
+
27
+        response = mock.Mock(text='<html></html>')
28
+        self.assertEqual(kickass.response(response), [])
29
+
30
+        html = """
31
+        <table cellpadding="0" cellspacing="0" class="data" style="width: 100%">
32
+            <tr class="firstr">
33
+                <th class="width100perc nopad">torrent name</th>
34
+                <th class="center">
35
+                    <a href="/search/test/?field=size&sorder=desc" rel="nofollow">size</a>
36
+                </th>
37
+                <th class="center"><span class="files">
38
+                    <a href="/search/test/?field=files_count&sorder=desc" rel="nofollow">files</a></span>
39
+                </th>
40
+                <th class="center"><span>
41
+                    <a href="/search/test/?field=time_add&sorder=desc" rel="nofollow">age</a></span>
42
+                </th>
43
+                <th class="center"><span class="seed">
44
+                    <a href="/search/test/?field=seeders&sorder=desc" rel="nofollow">seed</a></span>
45
+                </th>
46
+                <th class="lasttd nobr center">
47
+                    <a href="/search/test/?field=leechers&sorder=desc" rel="nofollow">leech</a>
48
+                </th>
49
+            </tr>
50
+            <tr class="even" id="torrent_test6478745">
51
+                <td>
52
+                    <div class="iaconbox center floatright">
53
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
54
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
55
+                            <i class="ka ka-comment"></i>
56
+                        </a>
57
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
58
+                            <i class="ka ka16 ka-verify ka-green"></i>
59
+                        </a>
60
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
61
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
62
+                        </a>
63
+                        <a title="Torrent magnet link"
64
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
65
+                            <i class="ka ka16 ka-magnet"></i>
66
+                        </a>
67
+                        <a title="Download torrent file"
68
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
69
+                            <i class="ka ka16 ka-arrow-down"></i>
70
+                        </a>
71
+                    </div>
72
+                    <div class="torrentname">
73
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
74
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
75
+                    <div class="markeredBlock torType txtType">
76
+                        <a href="/url.html" class="cellMainLink">
77
+                            <strong class="red">This should be the title</strong>
78
+                        </a>
79
+                        <span class="font11px lightgrey block">
80
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
81
+                            <a class="plain" href="/user/riri/">riri</a> in
82
+                            <span id="cat_6478745">
83
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
84
+                            </span>
85
+                        </span>
86
+                    </div>
87
+                </td>
88
+                <td class="nobr center">449 <span>bytes</span></td>
89
+                <td class="center">4</td>
90
+                <td class="center">2&nbsp;years</td>
91
+                <td class="green center">10</td>
92
+                <td class="red lasttd center">1</td>
93
+            </tr>
94
+        </table>
95
+        """
96
+        response = mock.Mock(text=html)
97
+        results = kickass.response(response)
98
+        self.assertEqual(type(results), list)
99
+        self.assertEqual(len(results), 1)
100
+        self.assertEqual(results[0]['title'], 'This should be the title')
101
+        self.assertEqual(results[0]['url'], 'https://kickass.so/url.html')
102
+        self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
103
+        self.assertEqual(results[0]['seed'], 10)
104
+        self.assertEqual(results[0]['leech'], 1)
105
+        self.assertEqual(results[0]['filesize'], 449)
106
+        self.assertEqual(results[0]['files'], 4)
107
+        self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETURL&dn=test')
108
+        self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/53917.torrent?title=test')
109
+
110
+        html = """
111
+        <table cellpadding="0" cellspacing="0" class="data" style="width: 100%">
112
+            <tr class="firstr">
113
+                <th class="width100perc nopad">torrent name</th>
114
+                <th class="center">
115
+                    <a href="/search/test/?field=size&sorder=desc" rel="nofollow">size</a>
116
+                </th>
117
+                <th class="center"><span class="files">
118
+                    <a href="/search/test/?field=files_count&sorder=desc" rel="nofollow">files</a></span>
119
+                </th>
120
+                <th class="center"><span>
121
+                    <a href="/search/test/?field=time_add&sorder=desc" rel="nofollow">age</a></span>
122
+                </th>
123
+                <th class="center"><span class="seed">
124
+                    <a href="/search/test/?field=seeders&sorder=desc" rel="nofollow">seed</a></span>
125
+                </th>
126
+                <th class="lasttd nobr center">
127
+                    <a href="/search/test/?field=leechers&sorder=desc" rel="nofollow">leech</a>
128
+                </th>
129
+            </tr>
130
+        </table>
131
+        """
132
+        response = mock.Mock(text=html)
133
+        results = kickass.response(response)
134
+        self.assertEqual(type(results), list)
135
+        self.assertEqual(len(results), 0)
136
+
137
+        html = """
138
+        <table cellpadding="0" cellspacing="0" class="data" style="width: 100%">
139
+            <tr class="firstr">
140
+                <th class="width100perc nopad">torrent name</th>
141
+                <th class="center">
142
+                    <a href="/search/test/?field=size&sorder=desc" rel="nofollow">size</a>
143
+                </th>
144
+                <th class="center"><span class="files">
145
+                    <a href="/search/test/?field=files_count&sorder=desc" rel="nofollow">files</a></span>
146
+                </th>
147
+                <th class="center"><span>
148
+                    <a href="/search/test/?field=time_add&sorder=desc" rel="nofollow">age</a></span>
149
+                </th>
150
+                <th class="center"><span class="seed">
151
+                    <a href="/search/test/?field=seeders&sorder=desc" rel="nofollow">seed</a></span>
152
+                </th>
153
+                <th class="lasttd nobr center">
154
+                    <a href="/search/test/?field=leechers&sorder=desc" rel="nofollow">leech</a>
155
+                </th>
156
+            </tr>
157
+            <tr class="even" id="torrent_test6478745">
158
+                <td>
159
+                    <div class="iaconbox center floatright">
160
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
161
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
162
+                            <i class="ka ka-comment"></i>
163
+                        </a>
164
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
165
+                            <i class="ka ka16 ka-verify ka-green"></i>
166
+                        </a>
167
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
168
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
169
+                        </a>
170
+                        <a title="Torrent magnet link"
171
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
172
+                            <i class="ka ka16 ka-magnet"></i>
173
+                        </a>
174
+                        <a title="Download torrent file"
175
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
176
+                            <i class="ka ka16 ka-arrow-down"></i>
177
+                        </a>
178
+                    </div>
179
+                    <div class="torrentname">
180
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
181
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
182
+                    <div class="markeredBlock torType txtType">
183
+                        <a href="/url.html" class="cellMainLink">
184
+                            <strong class="red">This should be the title</strong>
185
+                        </a>
186
+                        <span class="font11px lightgrey block">
187
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
188
+                            <a class="plain" href="/user/riri/">riri</a> in
189
+                            <span id="cat_6478745">
190
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
191
+                            </span>
192
+                        </span>
193
+                    </div>
194
+                </td>
195
+                <td class="nobr center">1 <span>KB</span></td>
196
+                <td class="center">4</td>
197
+                <td class="center">2&nbsp;years</td>
198
+                <td class="green center">10</td>
199
+                <td class="red lasttd center">1</td>
200
+            </tr>
201
+            <tr class="even" id="torrent_test6478745">
202
+                <td>
203
+                    <div class="iaconbox center floatright">
204
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
205
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
206
+                            <i class="ka ka-comment"></i>
207
+                        </a>
208
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
209
+                            <i class="ka ka16 ka-verify ka-green"></i>
210
+                        </a>
211
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
212
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
213
+                        </a>
214
+                        <a title="Torrent magnet link"
215
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
216
+                            <i class="ka ka16 ka-magnet"></i>
217
+                        </a>
218
+                        <a title="Download torrent file"
219
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
220
+                            <i class="ka ka16 ka-arrow-down"></i>
221
+                        </a>
222
+                    </div>
223
+                    <div class="torrentname">
224
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
225
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
226
+                    <div class="markeredBlock torType txtType">
227
+                        <a href="/url.html" class="cellMainLink">
228
+                            <strong class="red">This should be the title</strong>
229
+                        </a>
230
+                        <span class="font11px lightgrey block">
231
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
232
+                            <a class="plain" href="/user/riri/">riri</a> in
233
+                            <span id="cat_6478745">
234
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
235
+                            </span>
236
+                        </span>
237
+                    </div>
238
+                </td>
239
+                <td class="nobr center">1 <span>MB</span></td>
240
+                <td class="center">4</td>
241
+                <td class="center">2&nbsp;years</td>
242
+                <td class="green center">9</td>
243
+                <td class="red lasttd center">1</td>
244
+            </tr>
245
+            <tr class="even" id="torrent_test6478745">
246
+                <td>
247
+                    <div class="iaconbox center floatright">
248
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
249
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
250
+                            <i class="ka ka-comment"></i>
251
+                        </a>
252
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
253
+                            <i class="ka ka16 ka-verify ka-green"></i>
254
+                        </a>
255
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
256
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
257
+                        </a>
258
+                        <a title="Torrent magnet link"
259
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
260
+                            <i class="ka ka16 ka-magnet"></i>
261
+                        </a>
262
+                        <a title="Download torrent file"
263
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
264
+                            <i class="ka ka16 ka-arrow-down"></i>
265
+                        </a>
266
+                    </div>
267
+                    <div class="torrentname">
268
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
269
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
270
+                    <div class="markeredBlock torType txtType">
271
+                        <a href="/url.html" class="cellMainLink">
272
+                            <strong class="red">This should be the title</strong>
273
+                        </a>
274
+                        <span class="font11px lightgrey block">
275
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
276
+                            <a class="plain" href="/user/riri/">riri</a> in
277
+                            <span id="cat_6478745">
278
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
279
+                            </span>
280
+                        </span>
281
+                    </div>
282
+                </td>
283
+                <td class="nobr center">1 <span>GB</span></td>
284
+                <td class="center">4</td>
285
+                <td class="center">2&nbsp;years</td>
286
+                <td class="green center">8</td>
287
+                <td class="red lasttd center">1</td>
288
+            </tr>
289
+            <tr class="even" id="torrent_test6478745">
290
+                <td>
291
+                    <div class="iaconbox center floatright">
292
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
293
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
294
+                            <i class="ka ka-comment"></i>
295
+                        </a>
296
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
297
+                            <i class="ka ka16 ka-verify ka-green"></i>
298
+                        </a>
299
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
300
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
301
+                        </a>
302
+                        <a title="Torrent magnet link"
303
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
304
+                            <i class="ka ka16 ka-magnet"></i>
305
+                        </a>
306
+                        <a title="Download torrent file"
307
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
308
+                            <i class="ka ka16 ka-arrow-down"></i>
309
+                        </a>
310
+                    </div>
311
+                    <div class="torrentname">
312
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
313
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
314
+                    <div class="markeredBlock torType txtType">
315
+                        <a href="/url.html" class="cellMainLink">
316
+                            <strong class="red">This should be the title</strong>
317
+                        </a>
318
+                        <span class="font11px lightgrey block">
319
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
320
+                            <a class="plain" href="/user/riri/">riri</a> in
321
+                            <span id="cat_6478745">
322
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
323
+                            </span>
324
+                        </span>
325
+                    </div>
326
+                </td>
327
+                <td class="nobr center">1 <span>TB</span></td>
328
+                <td class="center">4</td>
329
+                <td class="center">2&nbsp;years</td>
330
+                <td class="green center">7</td>
331
+                <td class="red lasttd center">1</td>
332
+            </tr>
333
+            <tr class="even" id="torrent_test6478745">
334
+                <td>
335
+                    <div class="iaconbox center floatright">
336
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
337
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
338
+                            <i class="ka ka-comment"></i>
339
+                        </a>
340
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
341
+                            <i class="ka ka16 ka-verify ka-green"></i>
342
+                        </a>
343
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
344
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
345
+                        </a>
346
+                        <a title="Torrent magnet link"
347
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
348
+                            <i class="ka ka16 ka-magnet"></i>
349
+                        </a>
350
+                        <a title="Download torrent file"
351
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
352
+                            <i class="ka ka16 ka-arrow-down"></i>
353
+                        </a>
354
+                    </div>
355
+                    <div class="torrentname">
356
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
357
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
358
+                    <div class="markeredBlock torType txtType">
359
+                        <a href="/url.html" class="cellMainLink">
360
+                            <strong class="red">This should be the title</strong>
361
+                        </a>
362
+                        <span class="font11px lightgrey block">
363
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
364
+                            <a class="plain" href="/user/riri/">riri</a> in
365
+                            <span id="cat_6478745">
366
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
367
+                            </span>
368
+                        </span>
369
+                    </div>
370
+                </td>
371
+                <td class="nobr center">z <span>bytes</span></td>
372
+                <td class="center">r</td>
373
+                <td class="center">2&nbsp;years</td>
374
+                <td class="green center">a</td>
375
+                <td class="red lasttd center">t</td>
376
+            </tr>
377
+        </table>
378
+        """
379
+        response = mock.Mock(text=html)
380
+        results = kickass.response(response)
381
+        self.assertEqual(type(results), list)
382
+        self.assertEqual(len(results), 5)
383
+        self.assertEqual(results[0]['title'], 'This should be the title')
384
+        self.assertEqual(results[0]['url'], 'https://kickass.so/url.html')
385
+        self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
386
+        self.assertEqual(results[0]['seed'], 10)
387
+        self.assertEqual(results[0]['leech'], 1)
388
+        self.assertEqual(results[0]['files'], 4)
389
+        self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETURL&dn=test')
390
+        self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/53917.torrent?title=test')
391
+        self.assertEqual(results[0]['filesize'], 1024)
392
+        self.assertEqual(results[1]['filesize'], 1048576)
393
+        self.assertEqual(results[2]['filesize'], 1073741824)
394
+        self.assertEqual(results[3]['filesize'], 1099511627776)
395
+        self.assertEqual(results[4]['seed'], 0)
396
+        self.assertEqual(results[4]['leech'], 0)
397
+        self.assertEqual(results[4]['files'], None)
398
+        self.assertEqual(results[4]['filesize'], None)

+ 1
- 0
searx/tests/test_engines.py View File

@@ -9,6 +9,7 @@ from searx.tests.engines.test_digg import *  # noqa
9 9
 from searx.tests.engines.test_dummy import *  # noqa
10 10
 from searx.tests.engines.test_flickr import *  # noqa
11 11
 from searx.tests.engines.test_github import *  # noqa
12
+from searx.tests.engines.test_kickass import *  # noqa
12 13
 from searx.tests.engines.test_mixcloud import *  # noqa
13 14
 from searx.tests.engines.test_searchcode_code import *  # noqa
14 15
 from searx.tests.engines.test_searchcode_doc import *  # noqa