Merge pull request #728 from kvch/fix-kickass-torrents

fix kickass torrents engine

searx/engines/kickass.py

@@ -16,13 +16,14 @@ from urllib import quote
 from lxml import html
 from operator import itemgetter
 from searx.engines.xpath import extract_text
+from searx.utils import get_torrent_size, convert_str_to_int
 
 # engine dependent config
 categories = ['videos', 'music', 'files']
 paging = True
 
 # search-url
-url = 'https://kickass.to/'
+url = 'https://kickass.cd/'
 search_url = url + 'search/{search_term}/{pageno}/'
 
 # specific xpath variables
@@ -57,41 +58,16 @@ def response(resp):
         href = urljoin(url, link.attrib['href'])
         title = extract_text(link)
         content = escape(extract_text(result.xpath(content_xpath)))
-        seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]
-        leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]
-        filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0]
-        filesize_multiplier = result.xpath('.//td[contains(@class, "nobr")]//span/text()')[0]
-        files = result.xpath('.//td[contains(@class, "center")][2]/text()')[0]
-
-        # convert seed to int if possible
-        if seed.isdigit():
-            seed = int(seed)
-        else:
-            seed = 0
+        seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))
+        leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))
+        filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))
+        files = extract_text(result.xpath('.//td[contains(@class, "center")][2]'))
 
-        # convert leech to int if possible
-        if leech.isdigit():
-            leech = int(leech)
-        else:
-            leech = 0
-
-        # convert filesize to byte if possible
-        try:
-            filesize = float(filesize)
-
-            # convert filesize to byte
-            if filesize_multiplier == 'TB':
-                filesize = int(filesize * 1024 * 1024 * 1024 * 1024)
-            elif filesize_multiplier == 'GB':
-                filesize = int(filesize * 1024 * 1024 * 1024)
-            elif filesize_multiplier == 'MB':
-                filesize = int(filesize * 1024 * 1024)
-            elif filesize_multiplier == 'KB':
-                filesize = int(filesize * 1024)
-        except:
-            filesize = None
-
-        # convert files to int if possible
+        seed = convert_str_to_int(seed)
+        leech = convert_str_to_int(leech)
+
+        filesize, filesize_multiplier = filesize_info.split()
+        filesize = get_torrent_size(filesize, filesize_multiplier)
         if files.isdigit():
             files = int(files)
         else:

searx/settings.yml

@@ -307,6 +307,12 @@ engines:
     timeout : 6.0
     disabled : True
 
+  - name: kickass
+    engine : kickass
+    shortcut : kc
+    timeout : 4.0
+    disabled : True
+
   - name : microsoft academic
     engine : json_engine
     paging : True

searx/utils.py

@@ -252,12 +252,27 @@ def get_torrent_size(filesize, filesize_multiplier):
             filesize = int(filesize * 1024 * 1024)
         elif filesize_multiplier == 'KB':
             filesize = int(filesize * 1024)
+        elif filesize_multiplier == 'TiB':
+            filesize = int(filesize * 1000 * 1000 * 1000 * 1000)
+        elif filesize_multiplier == 'GiB':
+            filesize = int(filesize * 1000 * 1000 * 1000)
+        elif filesize_multiplier == 'MiB':
+            filesize = int(filesize * 1000 * 1000)
+        elif filesize_multiplier == 'KiB':
+            filesize = int(filesize * 1000)
     except:
         filesize = None
 
     return filesize
 
 
+def convert_str_to_int(number_str):
+    if number_str.isdigit():
+        return int(number_str)
+    else:
+        return 0
+
+
 def is_valid_lang(lang):
     is_abbr = (len(lang) == 2)
     if is_abbr:

tests/unit/engines/test_kickass.py

@@ -14,7 +14,7 @@ class TestKickassEngine(SearxTestCase):
         params = kickass.request(query, dicto)
         self.assertIn('url', params)
         self.assertIn(query, params['url'])
-        self.assertIn('kickass.to', params['url'])
+        self.assertIn('kickass.cd', params['url'])
         self.assertFalse(params['verify'])
 
     def test_response(self):
@@ -84,7 +84,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">449 <span>bytes</span></td>
+                <td class="nobr center">449 bytes</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">10</td>
@@ -97,7 +97,7 @@ class TestKickassEngine(SearxTestCase):
         self.assertEqual(type(results), list)
         self.assertEqual(len(results), 1)
         self.assertEqual(results[0]['title'], 'This should be the title')
-        self.assertEqual(results[0]['url'], 'https://kickass.to/url.html')
+        self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
         self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
         self.assertEqual(results[0]['seed'], 10)
         self.assertEqual(results[0]['leech'], 1)
@@ -191,7 +191,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>KB</span></td>
+                <td class="nobr center">1 KiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">10</td>
@@ -235,7 +235,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>MB</span></td>
+                <td class="nobr center">1 MiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">9</td>
@@ -279,7 +279,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>GB</span></td>
+                <td class="nobr center">1 GiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">8</td>
@@ -323,7 +323,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>TB</span></td>
+                <td class="nobr center">1 TiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">7</td>
@@ -367,7 +367,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">z <span>bytes</span></td>
+                <td class="nobr center">z bytes</td>
                 <td class="center">r</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">a</td>
@@ -380,17 +380,17 @@ class TestKickassEngine(SearxTestCase):
         self.assertEqual(type(results), list)
         self.assertEqual(len(results), 5)
         self.assertEqual(results[0]['title'], 'This should be the title')
-        self.assertEqual(results[0]['url'], 'https://kickass.to/url.html')
+        self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
         self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
         self.assertEqual(results[0]['seed'], 10)
         self.assertEqual(results[0]['leech'], 1)
         self.assertEqual(results[0]['files'], 4)
         self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETURL&dn=test')
         self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/53917.torrent?title=test')
-        self.assertEqual(results[0]['filesize'], 1024)
-        self.assertEqual(results[1]['filesize'], 1048576)
-        self.assertEqual(results[2]['filesize'], 1073741824)
-        self.assertEqual(results[3]['filesize'], 1099511627776)
+        self.assertEqual(results[0]['filesize'], 1000)
+        self.assertEqual(results[1]['filesize'], 1000000)
+        self.assertEqual(results[2]['filesize'], 1000000000)
+        self.assertEqual(results[3]['filesize'], 1000000000000)
         self.assertEqual(results[4]['seed'], 0)
         self.assertEqual(results[4]['leech'], 0)
         self.assertEqual(results[4]['files'], None)