Browse Source

Kickass' unit test

Cqoicebordel 10 years ago
parent
commit
8ea749d6ec
3 changed files with 403 additions and 4 deletions
  1. 4
    4
      searx/engines/kickass.py
  2. 398
    0
      searx/tests/engines/test_kickass.py
  3. 1
    0
      searx/tests/test_engines.py

+ 4
- 4
searx/engines/kickass.py View File

13
 from urllib import quote
13
 from urllib import quote
14
 from lxml import html
14
 from lxml import html
15
 from operator import itemgetter
15
 from operator import itemgetter
16
+from searx.engines.xpath import extract_text
16
 
17
 
17
 # engine dependent config
18
 # engine dependent config
18
 categories = ['videos', 'music', 'files']
19
 categories = ['videos', 'music', 'files']
56
     for result in search_res[1:]:
57
     for result in search_res[1:]:
57
         link = result.xpath('.//a[@class="cellMainLink"]')[0]
58
         link = result.xpath('.//a[@class="cellMainLink"]')[0]
58
         href = urljoin(url, link.attrib['href'])
59
         href = urljoin(url, link.attrib['href'])
59
-        title = ' '.join(link.xpath('.//text()'))
60
-        content = escape(html.tostring(result.xpath(content_xpath)[0],
61
-                                       method="text"))
60
+        title = extract_text(link)
61
+        content = escape(extract_text(result.xpath(content_xpath)))
62
         seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]
62
         seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]
63
         leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]
63
         leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]
64
         filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0]
64
         filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0]
88
                 filesize = int(filesize * 1024 * 1024 * 1024)
88
                 filesize = int(filesize * 1024 * 1024 * 1024)
89
             elif filesize_multiplier == 'MB':
89
             elif filesize_multiplier == 'MB':
90
                 filesize = int(filesize * 1024 * 1024)
90
                 filesize = int(filesize * 1024 * 1024)
91
-            elif filesize_multiplier == 'kb':
91
+            elif filesize_multiplier == 'KB':
92
                 filesize = int(filesize * 1024)
92
                 filesize = int(filesize * 1024)
93
         except:
93
         except:
94
             filesize = None
94
             filesize = None

+ 398
- 0
searx/tests/engines/test_kickass.py View File

1
+# -*- coding: utf-8 -*-
2
+from collections import defaultdict
3
+import mock
4
+from searx.engines import kickass
5
+from searx.testing import SearxTestCase
6
+
7
+
8
+class TestKickassEngine(SearxTestCase):
9
+
10
+    def test_request(self):
11
+        query = 'test_query'
12
+        dicto = defaultdict(dict)
13
+        dicto['pageno'] = 1
14
+        params = kickass.request(query, dicto)
15
+        self.assertIn('url', params)
16
+        self.assertIn(query, params['url'])
17
+        self.assertIn('kickass.so', params['url'])
18
+        self.assertIn('verify', params)
19
+        self.assertFalse(params['verify'])
20
+
21
+    def test_response(self):
22
+        self.assertRaises(AttributeError, kickass.response, None)
23
+        self.assertRaises(AttributeError, kickass.response, [])
24
+        self.assertRaises(AttributeError, kickass.response, '')
25
+        self.assertRaises(AttributeError, kickass.response, '[]')
26
+
27
+        response = mock.Mock(text='<html></html>')
28
+        self.assertEqual(kickass.response(response), [])
29
+
30
+        html = """
31
+        <table cellpadding="0" cellspacing="0" class="data" style="width: 100%">
32
+            <tr class="firstr">
33
+                <th class="width100perc nopad">torrent name</th>
34
+                <th class="center">
35
+                    <a href="/search/test/?field=size&sorder=desc" rel="nofollow">size</a>
36
+                </th>
37
+                <th class="center"><span class="files">
38
+                    <a href="/search/test/?field=files_count&sorder=desc" rel="nofollow">files</a></span>
39
+                </th>
40
+                <th class="center"><span>
41
+                    <a href="/search/test/?field=time_add&sorder=desc" rel="nofollow">age</a></span>
42
+                </th>
43
+                <th class="center"><span class="seed">
44
+                    <a href="/search/test/?field=seeders&sorder=desc" rel="nofollow">seed</a></span>
45
+                </th>
46
+                <th class="lasttd nobr center">
47
+                    <a href="/search/test/?field=leechers&sorder=desc" rel="nofollow">leech</a>
48
+                </th>
49
+            </tr>
50
+            <tr class="even" id="torrent_test6478745">
51
+                <td>
52
+                    <div class="iaconbox center floatright">
53
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
54
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
55
+                            <i class="ka ka-comment"></i>
56
+                        </a>
57
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
58
+                            <i class="ka ka16 ka-verify ka-green"></i>
59
+                        </a>
60
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
61
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
62
+                        </a>
63
+                        <a title="Torrent magnet link"
64
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
65
+                            <i class="ka ka16 ka-magnet"></i>
66
+                        </a>
67
+                        <a title="Download torrent file"
68
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
69
+                            <i class="ka ka16 ka-arrow-down"></i>
70
+                        </a>
71
+                    </div>
72
+                    <div class="torrentname">
73
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
74
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
75
+                    <div class="markeredBlock torType txtType">
76
+                        <a href="/url.html" class="cellMainLink">
77
+                            <strong class="red">This should be the title</strong>
78
+                        </a>
79
+                        <span class="font11px lightgrey block">
80
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
81
+                            <a class="plain" href="/user/riri/">riri</a> in
82
+                            <span id="cat_6478745">
83
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
84
+                            </span>
85
+                        </span>
86
+                    </div>
87
+                </td>
88
+                <td class="nobr center">449 <span>bytes</span></td>
89
+                <td class="center">4</td>
90
+                <td class="center">2&nbsp;years</td>
91
+                <td class="green center">10</td>
92
+                <td class="red lasttd center">1</td>
93
+            </tr>
94
+        </table>
95
+        """
96
+        response = mock.Mock(text=html)
97
+        results = kickass.response(response)
98
+        self.assertEqual(type(results), list)
99
+        self.assertEqual(len(results), 1)
100
+        self.assertEqual(results[0]['title'], 'This should be the title')
101
+        self.assertEqual(results[0]['url'], 'https://kickass.so/url.html')
102
+        self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
103
+        self.assertEqual(results[0]['seed'], 10)
104
+        self.assertEqual(results[0]['leech'], 1)
105
+        self.assertEqual(results[0]['filesize'], 449)
106
+        self.assertEqual(results[0]['files'], 4)
107
+        self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETURL&dn=test')
108
+        self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/53917.torrent?title=test')
109
+
110
+        html = """
111
+        <table cellpadding="0" cellspacing="0" class="data" style="width: 100%">
112
+            <tr class="firstr">
113
+                <th class="width100perc nopad">torrent name</th>
114
+                <th class="center">
115
+                    <a href="/search/test/?field=size&sorder=desc" rel="nofollow">size</a>
116
+                </th>
117
+                <th class="center"><span class="files">
118
+                    <a href="/search/test/?field=files_count&sorder=desc" rel="nofollow">files</a></span>
119
+                </th>
120
+                <th class="center"><span>
121
+                    <a href="/search/test/?field=time_add&sorder=desc" rel="nofollow">age</a></span>
122
+                </th>
123
+                <th class="center"><span class="seed">
124
+                    <a href="/search/test/?field=seeders&sorder=desc" rel="nofollow">seed</a></span>
125
+                </th>
126
+                <th class="lasttd nobr center">
127
+                    <a href="/search/test/?field=leechers&sorder=desc" rel="nofollow">leech</a>
128
+                </th>
129
+            </tr>
130
+        </table>
131
+        """
132
+        response = mock.Mock(text=html)
133
+        results = kickass.response(response)
134
+        self.assertEqual(type(results), list)
135
+        self.assertEqual(len(results), 0)
136
+
137
+        html = """
138
+        <table cellpadding="0" cellspacing="0" class="data" style="width: 100%">
139
+            <tr class="firstr">
140
+                <th class="width100perc nopad">torrent name</th>
141
+                <th class="center">
142
+                    <a href="/search/test/?field=size&sorder=desc" rel="nofollow">size</a>
143
+                </th>
144
+                <th class="center"><span class="files">
145
+                    <a href="/search/test/?field=files_count&sorder=desc" rel="nofollow">files</a></span>
146
+                </th>
147
+                <th class="center"><span>
148
+                    <a href="/search/test/?field=time_add&sorder=desc" rel="nofollow">age</a></span>
149
+                </th>
150
+                <th class="center"><span class="seed">
151
+                    <a href="/search/test/?field=seeders&sorder=desc" rel="nofollow">seed</a></span>
152
+                </th>
153
+                <th class="lasttd nobr center">
154
+                    <a href="/search/test/?field=leechers&sorder=desc" rel="nofollow">leech</a>
155
+                </th>
156
+            </tr>
157
+            <tr class="even" id="torrent_test6478745">
158
+                <td>
159
+                    <div class="iaconbox center floatright">
160
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
161
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
162
+                            <i class="ka ka-comment"></i>
163
+                        </a>
164
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
165
+                            <i class="ka ka16 ka-verify ka-green"></i>
166
+                        </a>
167
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
168
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
169
+                        </a>
170
+                        <a title="Torrent magnet link"
171
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
172
+                            <i class="ka ka16 ka-magnet"></i>
173
+                        </a>
174
+                        <a title="Download torrent file"
175
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
176
+                            <i class="ka ka16 ka-arrow-down"></i>
177
+                        </a>
178
+                    </div>
179
+                    <div class="torrentname">
180
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
181
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
182
+                    <div class="markeredBlock torType txtType">
183
+                        <a href="/url.html" class="cellMainLink">
184
+                            <strong class="red">This should be the title</strong>
185
+                        </a>
186
+                        <span class="font11px lightgrey block">
187
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
188
+                            <a class="plain" href="/user/riri/">riri</a> in
189
+                            <span id="cat_6478745">
190
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
191
+                            </span>
192
+                        </span>
193
+                    </div>
194
+                </td>
195
+                <td class="nobr center">1 <span>KB</span></td>
196
+                <td class="center">4</td>
197
+                <td class="center">2&nbsp;years</td>
198
+                <td class="green center">10</td>
199
+                <td class="red lasttd center">1</td>
200
+            </tr>
201
+            <tr class="even" id="torrent_test6478745">
202
+                <td>
203
+                    <div class="iaconbox center floatright">
204
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
205
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
206
+                            <i class="ka ka-comment"></i>
207
+                        </a>
208
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
209
+                            <i class="ka ka16 ka-verify ka-green"></i>
210
+                        </a>
211
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
212
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
213
+                        </a>
214
+                        <a title="Torrent magnet link"
215
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
216
+                            <i class="ka ka16 ka-magnet"></i>
217
+                        </a>
218
+                        <a title="Download torrent file"
219
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
220
+                            <i class="ka ka16 ka-arrow-down"></i>
221
+                        </a>
222
+                    </div>
223
+                    <div class="torrentname">
224
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
225
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
226
+                    <div class="markeredBlock torType txtType">
227
+                        <a href="/url.html" class="cellMainLink">
228
+                            <strong class="red">This should be the title</strong>
229
+                        </a>
230
+                        <span class="font11px lightgrey block">
231
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
232
+                            <a class="plain" href="/user/riri/">riri</a> in
233
+                            <span id="cat_6478745">
234
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
235
+                            </span>
236
+                        </span>
237
+                    </div>
238
+                </td>
239
+                <td class="nobr center">1 <span>MB</span></td>
240
+                <td class="center">4</td>
241
+                <td class="center">2&nbsp;years</td>
242
+                <td class="green center">9</td>
243
+                <td class="red lasttd center">1</td>
244
+            </tr>
245
+            <tr class="even" id="torrent_test6478745">
246
+                <td>
247
+                    <div class="iaconbox center floatright">
248
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
249
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
250
+                            <i class="ka ka-comment"></i>
251
+                        </a>
252
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
253
+                            <i class="ka ka16 ka-verify ka-green"></i>
254
+                        </a>
255
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
256
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
257
+                        </a>
258
+                        <a title="Torrent magnet link"
259
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
260
+                            <i class="ka ka16 ka-magnet"></i>
261
+                        </a>
262
+                        <a title="Download torrent file"
263
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
264
+                            <i class="ka ka16 ka-arrow-down"></i>
265
+                        </a>
266
+                    </div>
267
+                    <div class="torrentname">
268
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
269
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
270
+                    <div class="markeredBlock torType txtType">
271
+                        <a href="/url.html" class="cellMainLink">
272
+                            <strong class="red">This should be the title</strong>
273
+                        </a>
274
+                        <span class="font11px lightgrey block">
275
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
276
+                            <a class="plain" href="/user/riri/">riri</a> in
277
+                            <span id="cat_6478745">
278
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
279
+                            </span>
280
+                        </span>
281
+                    </div>
282
+                </td>
283
+                <td class="nobr center">1 <span>GB</span></td>
284
+                <td class="center">4</td>
285
+                <td class="center">2&nbsp;years</td>
286
+                <td class="green center">8</td>
287
+                <td class="red lasttd center">1</td>
288
+            </tr>
289
+            <tr class="even" id="torrent_test6478745">
290
+                <td>
291
+                    <div class="iaconbox center floatright">
292
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
293
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
294
+                            <i class="ka ka-comment"></i>
295
+                        </a>
296
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
297
+                            <i class="ka ka16 ka-verify ka-green"></i>
298
+                        </a>
299
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
300
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
301
+                        </a>
302
+                        <a title="Torrent magnet link"
303
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
304
+                            <i class="ka ka16 ka-magnet"></i>
305
+                        </a>
306
+                        <a title="Download torrent file"
307
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
308
+                            <i class="ka ka16 ka-arrow-down"></i>
309
+                        </a>
310
+                    </div>
311
+                    <div class="torrentname">
312
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
313
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
314
+                    <div class="markeredBlock torType txtType">
315
+                        <a href="/url.html" class="cellMainLink">
316
+                            <strong class="red">This should be the title</strong>
317
+                        </a>
318
+                        <span class="font11px lightgrey block">
319
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
320
+                            <a class="plain" href="/user/riri/">riri</a> in
321
+                            <span id="cat_6478745">
322
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
323
+                            </span>
324
+                        </span>
325
+                    </div>
326
+                </td>
327
+                <td class="nobr center">1 <span>TB</span></td>
328
+                <td class="center">4</td>
329
+                <td class="center">2&nbsp;years</td>
330
+                <td class="green center">7</td>
331
+                <td class="red lasttd center">1</td>
332
+            </tr>
333
+            <tr class="even" id="torrent_test6478745">
334
+                <td>
335
+                    <div class="iaconbox center floatright">
336
+                        <a rel="6478745,0" class="icommentjs icon16" href="/test-t6478745.html#comment">
337
+                            <em style="font-size: 12px; margin: 0 4px 0 4px;" class="iconvalue">3</em>
338
+                            <i class="ka ka-comment"></i>
339
+                        </a>
340
+                        <a class="iverify icon16" href="/test-t6478745.html" title="Verified Torrent">
341
+                            <i class="ka ka16 ka-verify ka-green"></i>
342
+                        </a>
343
+                        <a href="#" onclick="_scq.push([]); return false;" class="partner1Button idownload icon16">
344
+                            <i class="ka ka16 ka-arrow-down partner1Button"></i>
345
+                        </a>
346
+                        <a title="Torrent magnet link"
347
+                            href="magnet:?xt=urn:btih:MAGNETURL&dn=test" class="imagnet icon16">
348
+                            <i class="ka ka16 ka-magnet"></i>
349
+                        </a>
350
+                        <a title="Download torrent file"
351
+                            href="http://torcache.net/torrent/53917.torrent?title=test" class="idownload icon16">
352
+                            <i class="ka ka16 ka-arrow-down"></i>
353
+                        </a>
354
+                    </div>
355
+                    <div class="torrentname">
356
+                    <a href="/test-t6478745.html" class="torType txtType"></a>
357
+                    <a href="/test-t6478745.html" class="normalgrey font12px plain bold"></a>
358
+                    <div class="markeredBlock torType txtType">
359
+                        <a href="/url.html" class="cellMainLink">
360
+                            <strong class="red">This should be the title</strong>
361
+                        </a>
362
+                        <span class="font11px lightgrey block">
363
+                            Posted by <i class="ka ka-verify" style="font-size: 16px;color:orange;"></i>
364
+                            <a class="plain" href="/user/riri/">riri</a> in
365
+                            <span id="cat_6478745">
366
+                                <strong><a href="/other/">Other</a> > <a href="/unsorted/">Unsorted</a></strong>
367
+                            </span>
368
+                        </span>
369
+                    </div>
370
+                </td>
371
+                <td class="nobr center">z <span>bytes</span></td>
372
+                <td class="center">r</td>
373
+                <td class="center">2&nbsp;years</td>
374
+                <td class="green center">a</td>
375
+                <td class="red lasttd center">t</td>
376
+            </tr>
377
+        </table>
378
+        """
379
+        response = mock.Mock(text=html)
380
+        results = kickass.response(response)
381
+        self.assertEqual(type(results), list)
382
+        self.assertEqual(len(results), 5)
383
+        self.assertEqual(results[0]['title'], 'This should be the title')
384
+        self.assertEqual(results[0]['url'], 'https://kickass.so/url.html')
385
+        self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
386
+        self.assertEqual(results[0]['seed'], 10)
387
+        self.assertEqual(results[0]['leech'], 1)
388
+        self.assertEqual(results[0]['files'], 4)
389
+        self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETURL&dn=test')
390
+        self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/53917.torrent?title=test')
391
+        self.assertEqual(results[0]['filesize'], 1024)
392
+        self.assertEqual(results[1]['filesize'], 1048576)
393
+        self.assertEqual(results[2]['filesize'], 1073741824)
394
+        self.assertEqual(results[3]['filesize'], 1099511627776)
395
+        self.assertEqual(results[4]['seed'], 0)
396
+        self.assertEqual(results[4]['leech'], 0)
397
+        self.assertEqual(results[4]['files'], None)
398
+        self.assertEqual(results[4]['filesize'], None)

+ 1
- 0
searx/tests/test_engines.py View File

9
 from searx.tests.engines.test_dummy import *  # noqa
9
 from searx.tests.engines.test_dummy import *  # noqa
10
 from searx.tests.engines.test_flickr import *  # noqa
10
 from searx.tests.engines.test_flickr import *  # noqa
11
 from searx.tests.engines.test_github import *  # noqa
11
 from searx.tests.engines.test_github import *  # noqa
12
+from searx.tests.engines.test_kickass import *  # noqa
12
 from searx.tests.engines.test_mixcloud import *  # noqa
13
 from searx.tests.engines.test_mixcloud import *  # noqa
13
 from searx.tests.engines.test_searchcode_code import *  # noqa
14
 from searx.tests.engines.test_searchcode_code import *  # noqa
14
 from searx.tests.engines.test_searchcode_doc import *  # noqa
15
 from searx.tests.engines.test_searchcode_doc import *  # noqa