瀏覽代碼

Merge pull request #677 from pydo/feature/seedpeer-engine-integration

Feature/seedpeer engine integration
Adam Tauber 8 年之前
父節點
當前提交
e7d005c621
共有 5 個檔案被更改,包括 246 行新增0 行删除
  1. 1
    0
      AUTHORS.rst
  2. 78
    0
      searx/engines/seedpeer.py
  3. 6
    0
      searx/settings.yml
  4. 110
    0
      tests/unit/engines/seedpeer_fixture.html
  5. 51
    0
      tests/unit/engines/test_seedpeer.py

+ 1
- 0
AUTHORS.rst 查看文件

@@ -58,3 +58,4 @@ generally made searx better:
58 58
 - marc @a01200356
59 59
 - Harry Wood @harry-wood
60 60
 - Thomas Renard @threnard
61
+- Pydo `<https://github.com/pydo>`_

+ 78
- 0
searx/engines/seedpeer.py 查看文件

@@ -0,0 +1,78 @@
1
+#  Seedpeer (Videos, Music, Files)
2
+#
3
+# @website     http://seedpeer.eu
4
+# @provide-api no (nothing found)
5
+#
6
+# @using-api   no
7
+# @results     HTML (using search portal)
8
+# @stable      yes (HTML can change)
9
+# @parse       url, title, content, seed, leech, magnetlink
10
+
11
+from urlparse import urljoin
12
+from cgi import escape
13
+from urllib import quote
14
+from lxml import html
15
+from operator import itemgetter
16
+from searx.engines.xpath import extract_text
17
+
18
+
19
+url = 'http://www.seedpeer.eu/'
20
+search_url = url + 'search/{search_term}/7/{page_no}.html'
21
+# specific xpath variables
22
+torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
23
+alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
24
+title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
25
+alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
26
+seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
27
+alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
28
+peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
29
+alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
30
+age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
31
+alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
32
+size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
33
+alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
34
+
35
+
36
+# do search-request
37
+def request(query, params):
38
+    params['url'] = search_url.format(search_term=quote(query),
39
+                                      page_no=params['pageno'] - 1)
40
+    return params
41
+
42
+
43
+# get response from search-request
44
+def response(resp):
45
+    results = []
46
+    dom = html.fromstring(resp.text)
47
+    torrent_links = dom.xpath(torrent_xpath)
48
+    if len(torrent_links) > 0:
49
+        seeds = dom.xpath(seeds_xpath)
50
+        peers = dom.xpath(peers_xpath)
51
+        titles = dom.xpath(title_xpath)
52
+        sizes = dom.xpath(size_xpath)
53
+        ages = dom.xpath(age_xpath)
54
+    else:  # under ~5 results uses a different xpath
55
+        torrent_links = dom.xpath(alternative_torrent_xpath)
56
+        seeds = dom.xpath(alternative_seeds_xpath)
57
+        peers = dom.xpath(alternative_peers_xpath)
58
+        titles = dom.xpath(alternative_title_xpath)
59
+        sizes = dom.xpath(alternative_size_xpath)
60
+        ages = dom.xpath(alternative_age_xpath)
61
+    # return empty array if nothing is found
62
+    if not torrent_links:
63
+        return []
64
+
65
+    # parse results
66
+    for index, result in enumerate(torrent_links):
67
+        link = result.attrib.get('href')
68
+        href = urljoin(url, link)
69
+        results.append({'url': href,
70
+                        'title': titles[index].text_content(),
71
+                        'content': '{}, {}'.format(sizes[index], ages[index]),
72
+                        'seed': seeds[index],
73
+                        'leech': peers[index],
74
+
75
+                        'template': 'torrent.html'})
76
+
77
+    # return results sorted by seeder
78
+    return sorted(results, key=itemgetter('seed'), reverse=True)

+ 6
- 0
searx/settings.yml 查看文件

@@ -495,6 +495,12 @@ engines:
495 495
     timeout: 6.0
496 496
     categories : science
497 497
 
498
+  - name : seedpeer
499
+    engine : seedpeer
500
+    shortcut: speu
501
+    categories: files, music, videos
502
+    disabled: True
503
+
498 504
   - name : dictzone
499 505
     engine : dictzone
500 506
     shortcut : dc

+ 110
- 0
tests/unit/engines/seedpeer_fixture.html 查看文件

@@ -0,0 +1,110 @@
1
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
2
+        <html xmlns="http://www.w3.org/1999/xhtml">
3
+        <head>
4
+        </head>
5
+        <body>
6
+            <div id="header">
7
+                <div id="whoIsYou">
8
+                        <a href="/lang.php"><small>SeedPeer in your own language?</small></a>&nbsp;&nbsp;&nbsp;<a href="http://www.seedpeer.eu"><img src="/images/flags/uk.gif" width="16px" alt="Torrents EN" /></a> <a href="http://spanish.seedpeer.eu"><img src="/images/flags/es.gif" width="16px" alt="Torrents ES" /></a> <a href="http://german.seedpeer.eu"><img src="/images/flags/de.gif" width="16px" alt="Torrents DE" /></a> <a href="http://french.seedpeer.eu"><img src="/images/flags/fr.gif" width="16px" alt="Torrents FR" /></a> <a href="http://portuguese.seedpeer.eu"><img src="/images/flags/pt.gif" width="16px" alt="Torrents Portuguese" /></a> <a href="http://swedish.seedpeer.eu"><img src="/images/flags/se.gif" width="16px" alt="Torrents Sweden" /></a>
9
+                        </div>
10
+
11
+                <script type="text/javascript">
12
+                whoIsYou();
13
+                </script>
14
+                    <div id="search">
15
+                        <form action="/search.php" method="get">
16
+                            <input id="topsearchbar" name="search" value="narcos season 2" />
17
+                            <input type="submit" class="searchbutton" value="Torrents" />
18
+                            <input style="color:#000" type="submit" class="searchbutton" name="usenet" value="Usenet Binaries" />
19
+                        </form>
20
+                        <div id="suggestion"></div>
21
+                    </div>
22
+                    <div id="logo"><a href="/"><img src="/images/logo2.gif" alt="Seedpeer homepage" width="415" height="143" /></a></div>
23
+                    <div id="subtext"><a href="/">Home</a> &gt; <a href="/search.html">Torrent search</a> &gt; Narcos season 2 | page 1</div>
24
+            </div>
25
+        <div id="nav">
26
+            <ul>
27
+        <!--
28
+                <li><font style="color:red;font-size:9px;font-weight:bold;">NEW</font><a title="Download TOP Games for FREE" rel="nofollow" href="http://www.bigrebelads.com/affiliate/index?ref=9301" target="_blank">FREE Games</a></li>
29
+
30
+        -->
31
+                <li style="border-left:none" id="categories"><a title="Browse Torrent Categories" href="/browse.html">Categories</a>
32
+                    <ul>
33
+                        <li><a title="Browse Anime Torrents" href="/browse.html#6">Anime</a></li>
34
+                        <li><a title="Browse Game Torrents" href="/browse.html#4">Games</a></li>
35
+                        <li><a title="Browse Movie Torrents" href="/browse.html#1">Movies</a></li>
36
+                        <li><a title="Browse Music Torrents" href="/browse.html#3">Music</a></li>
37
+                        <li><a title="Browse Software Torrents" href="/browse.html#5">Software</a></li>
38
+                        <li><a title="Browse TV Torrents" href="/browse.html#2">TV Shows</a></li>
39
+                        <li><a title="Browse Other Torrents" href="/browse.html#7">Others</a></li>
40
+                    </ul>
41
+                    </li>
42
+                        <li><a title="Upload A Torrents" href="/upload.html">Upload torrent</a></li>
43
+                        <li id="verified"><a title="Verified Torrents" href="/verified.html">Verified</a></li>
44
+                        <li id="searchoptions"><a title="Search Torrents" href="/search.html">Torrent search</a></li>
45
+                        <li id="newsgroups"><a style="color:#212b3e" title="News Groups" href="/usenet.html">Usenet Binaries</a></li>
46
+                        <li id="about" style="border-right:none"><a rel="nofollow" href="/faq.html">About Us</a>
47
+                    <ul>
48
+                        <li><a title="SeedPeer Statistics" href="/stats.html">Statistics</a></li>
49
+                        <li><a title="Contact Us" href="/contact.html">Contact</a></li>
50
+                        <li><a title="Frequently  Asked Questions" href="/faq.html">FAQ</a></li>
51
+                        <li><a title="SeedPeer API" href="http://api.seedpeer.eu">Our API</a></li>
52
+                        <li><a title="SeedPeer Blog" href="/blog">Blog</a></li>
53
+                    </ul>
54
+                    </li>
55
+                    <!--<li><a href="/toolbar.php">Our Toolbar</a></li>-->
56
+                </ul>
57
+            <div class="clear"></div>
58
+        </div>
59
+        <div id="body"><div id="pageTop"></div>
60
+        <div id="headerbox"><h1>Verified <font class="colored">Narcos season 2</font> torrents</h1></div><table width="100%"><tr><th>
61
+        <span style="float:right">
62
+        <a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
63
+        <a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
64
+        </span>
65
+        <a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th><td class="tableAd" rowspan="6"><iframe src="http://creative.wwwpromoter.com/13689?d=300x250" width="300" height="250" style="border: none;" frameborder="0" scrolling="no"></iframe></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_1" href="" data-tad="431726" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Full Version</a></td><td class="right">20 hours</td><td class="right">681.3 MB</td><td class="right"><font color="green">28</font> </td><td class="right"><font color="navy">654</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_2" href="" data-tad="431727" data-url="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Trusted Source</a></td><td class="right">12 hours</td><td class="right">787.1 MB</td><td class="right"><font color="green">64</font> </td><td class="right"><font color="navy">220</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_3" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Full Narcos season 2 Download</strong></a> <small><a class="pblink" id="pblink_table_item_4" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Usenet</a></small></td><td class="right">24 hours</td><td class="right">775.5 MB</td><td class="right"><font color="green">60</font> </td><td class="right"><font color="navy">236</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_5" href="" data-tad="431730" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014 - DIRECT STREAMING</a> <small><a class="pblink" id="pblink_table_item_6" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">17 hours</td><td class="right">654.1 MB</td><td class="right"><font color="green">2</font> </td><td class="right"><font color="navy">391</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_7" href="" data-tad="431731" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014</a> <small><a class="pblink" id="pblink_table_item_8" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">20 hours</td><td class="right">754.5 MB</td><td class="right"><font color="green">21</font> </td><td class="right"><font color="navy">919</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr></table><br /><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=728x90' width='728' height='90' style='border: none;' frameborder='0' scrolling='no'></iframe><center><span style="float:right;margin:1em .2em 0 0"><a title="Download at the speed of your connection" href="/usenet.php?search=narcos+season+2"><img src="/images/dlf.gif" alt="Search Binaries" /></a></span><div style="margin-bottom:1em;margin-right:290px" id="headerbox"><h1><a href="/searchfeed/narcos+season+2.xml" target="_blank" title="SeedPeer RSS Torrent Search Feed fornarcos season 2"><img src="/images/feedIcon.png" border="0" /></a>&nbsp;2 <font class="colored">Narcos season 2</font> Torrents were found</h1></div><table width="100%"><tr><th>
66
+        <span style="float:right">
67
+        <a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
68
+        <a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
69
+        </span>
70
+        <a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th></tr><tr class=""><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong>%20 torrent&u=http://seedpeer.seedpeer.eu/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong> <small><a href="/browse.html#11686840"></a></small></a></td><td class="right">19 hours</td><td class="right">4.39 GB</td><td class="right"><font color="green">715</font> </td><td class="right"><font color="navy">183</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr><tr class="tdark"><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG%20 torrent&u=http://seedpeer.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG <small><a href="/browse.html#11685972"></a></small></a></td><td class="right">1 day</td><td class="right">2.48 GB</td><td class="right"><font color="green">861</font> </td><td class="right"><font color="navy">332</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr></table><div id="headerbox"><h1>Related searches for: <font class="colored">Narcos season 2</font></h1></div><div id="search_suggestions"><br />Other suggested searches: </div><br /><a href="http://torrentz2.eu/search?f=narcos-season-2">Search for "narcos-season-2" on Torrentz2.eu</a><br /><a href="http://torrent-finder.info/show.php?q=narcos-season-2">Search for "narcos-season-2" on Torrent-Finder</a><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe></center><div id="footer">
71
+        <table width="100%">
72
+        <tr>
73
+        <td width="30%">
74
+        <h2>Torrents Download</h2>
75
+        <a href="/">Torrent search</a><br />
76
+        <a href="/browse.html">Browse categories</a><br />
77
+        <a href="/verified.html">Verified Torrents</a><br />
78
+        <a href="/order-date.html">Today's torrents</a><br />
79
+        <a href="/yesterday.html">Yesterday's torrents</a><br />
80
+        <a href="/stats.html">Statistics</a><br />
81
+        <br />
82
+        <a href="/faq.html#copyright"><strong>Copyright & Removal</strong></a>
83
+        </td>
84
+        <td width="30%"><h2>Cool Stuff</h2>
85
+        <a href="/promotional.php">Promotional</a><br />
86
+        <a href="/contact.html">Advertising Information</a><br />
87
+        <strong><a href="/plugins.php" title="Add a search plugin to Firefox or Internet Explorer">Search Plugin <span style="color:red">*</span></a></strong><br />
88
+        <a href="http://www.utorrent.com">&micro;Torrent Client</a><br />
89
+        <a href="/blog">Seedpeer Blog</a><br />
90
+        </td>
91
+        <td width="30%"><h2>Links</h2>
92
+         <a href="http://www.sumotorrent.com" target="_blank"><strong>SumoTorrent</strong></a><br />
93
+         <a href="http://www.torrent-finder.info" target="_blank"><strong>Torrent Finder</strong></a><br />
94
+         <a href="http://www.torrentpond.com" target="_blank"><strong>TorrentPond</strong></a><br />
95
+         <a href="https://www.limetorrents.cc" target="_blank">LimeTorrents.cc</a><br />
96
+         <a href="http://www.torrents.to/" target="_blank">Torrents.to</a><br />
97
+         <a href="http://www.torrentfunk.com" target="_blank">TorrentFunk</a><br />
98
+         <a href="https://monova.org" target="_blank">Monova</a><br />
99
+         <a href="http://www.torrentroom.com" target="_blank">TorrentRoom</a><br />
100
+         <a href="http://www.katcr.co/" target="_blank">Kickass Torrents Community</a><br />
101
+        </td>
102
+        <td width="10%"><div id="bottomlogo"></div></td>
103
+        </tr>
104
+        </table>
105
+        <br />
106
+        <br />
107
+        </div>
108
+        </div>
109
+        </body>
110
+        </html>

+ 51
- 0
tests/unit/engines/test_seedpeer.py 查看文件

@@ -0,0 +1,51 @@
1
+import mock
2
+from collections import defaultdict
3
+from searx.engines import seedpeer
4
+from searx.testing import SearxTestCase
5
+from datetime import datetime
6
+
7
+
8
+class TestSeedPeerEngine(SearxTestCase):
9
+
10
+    html = ''
11
+    with open('./tests/unit/engines/seedpeer_fixture.html') as fixture:
12
+        html += fixture.read()
13
+
14
+    def test_request(self):
15
+        query = 'test_query'
16
+        dicto = defaultdict(dict)
17
+        dicto['pageno'] = 1
18
+        params = seedpeer.request(query, dicto)
19
+        self.assertIn('url', params)
20
+        self.assertIn(query, params['url'])
21
+        self.assertIn('seedpeer.eu', params['url'])
22
+
23
+    def test_response_raises_attr_error_on_empty_response(self):
24
+        self.assertRaises(AttributeError, seedpeer.response, None)
25
+        self.assertRaises(AttributeError, seedpeer.response, [])
26
+        self.assertRaises(AttributeError, seedpeer.response, '')
27
+        self.assertRaises(AttributeError, seedpeer.response, '[]')
28
+
29
+    def test_response_returns_empty_list(self):
30
+        response = mock.Mock(text='<html></html>')
31
+        self.assertEqual(seedpeer.response(response), [])
32
+
33
+    def test_response_returns_all_results(self):
34
+        response = mock.Mock(text=self.html)
35
+        results = seedpeer.response(response)
36
+        self.assertTrue(isinstance(results, list))
37
+        self.assertEqual(len(results), 2)
38
+
39
+    def test_response_returns_correct_results(self):
40
+        response = mock.Mock(text=self.html)
41
+        results = seedpeer.response(response)
42
+        self.assertEqual(
43
+            results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG '
44
+        )
45
+        self.assertEqual(
46
+            results[0]['url'],
47
+            'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html'
48
+        )
49
+        self.assertEqual(results[0]['content'], '2.48 GB, 1 day')
50
+        self.assertEqual(results[0]['seed'], '861')
51
+        self.assertEqual(results[0]['leech'], '332')