Merge pull request #89 from pointhi/engines

update search engines and add comments to it

searx/engines/bing.py

 base_url = 'https://www.bing.com/'
 search_string = 'search?{query}&first={offset}'
 
+
 # do search-request
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1

searx/engines/bing_images.py

 base_url = 'https://www.bing.com/'
 search_string = 'images/search?{query}&count=10&first={offset}'
 
+
 # do search-request
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1

searx/engines/bing_news.py

 base_url = 'https://www.bing.com/'
 search_string = 'news/search?{query}&first={offset}'
 
+
 # do search-request
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1

searx/engines/deviantart.py

+## Deviantart (Images)
+# 
+# @website     https://www.deviantart.com/
+# @provide-api yes (https://www.deviantart.com/developers/) (RSS)
+# 
+# @using-api   no (TODO, rewrite to api)
+# @results     HTML
+# @stable      no (HTML can change)
+# @parse       url, title, thumbnail, img_src
+#
+# @todo        rewrite to api
+
 from urllib import urlencode
 from urlparse import urljoin
 from lxml import html
 
+# engine dependent config
 categories = ['images']
+paging = True
 
+# search-url
 base_url = 'https://www.deviantart.com/'
 search_url = base_url+'search?offset={offset}&{query}'
 
-paging = True
-
 
+# do search-request
 def request(query, params):
     offset = (params['pageno'] - 1) * 24
+
     params['url'] = search_url.format(offset=offset,
                                       query=urlencode({'q': query}))
+
     return params
 
 
+# get response from search-request
 def response(resp):
     results = []
+
+    # return empty array if a redirection code is returned
     if resp.status_code == 302:
-        return results
+        return []
+
     dom = html.fromstring(resp.text)
+
+    # parse results
     for result in dom.xpath('//div[contains(@class, "tt-a tt-fh")]'):
         link = result.xpath('.//a[contains(@class, "thumb")]')[0]
         url = urljoin(base_url, link.attrib.get('href'))
         title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]')  # noqa
         title = ''.join(title_links[0].xpath('.//text()'))
         img_src = link.xpath('.//img')[0].attrib['src']
+
+        # append result
         results.append({'url': url,
                         'title': title,
                         'img_src': img_src,
                         'template': 'images.html'})
+
+    # return results
     return results

searx/engines/duckduckgo.py

+## DuckDuckGo (Web)
+# 
+# @website     https://duckduckgo.com/
+# @provide-api yes (https://duckduckgo.com/api), but not all results from search-site
+# 
+# @using-api   no
+# @results     HTML (using search portal)
+# @stable      no (HTML can change)
+# @parse       url, title, content
+#
+# @todo        rewrite to api
+# @todo        language support (the current used site does not support language-change)
+
 from urllib import urlencode
 from lxml.html import fromstring
 from searx.utils import html_to_text
 
-url = 'https://duckduckgo.com/html?{query}&s={offset}'
+# engine dependent config
+categories = ['general']
+paging = True
 locale = 'us-en'
 
+# search-url
+url = 'https://duckduckgo.com/html?{query}&s={offset}'
+
+# specific xpath variables
+result_xpath = '//div[@class="results_links results_links_deep web-result"]'  # noqa
+url_xpath = './/a[@class="large"]/@href'
+title_xpath = './/a[@class="large"]//text()'
+content_xpath = './/div[@class="snippet"]//text()'
 
+
+# do search-request
 def request(query, params):
     offset = (params['pageno'] - 1) * 30
-    q = urlencode({'q': query,
-                   'l': locale})
-    params['url'] = url.format(query=q, offset=offset)
+
+    params['url'] = url.format(
+        query=urlencode({'q': query, 'l': locale}),
+        offset=offset)
+
     return params
 
 
+# get response from search-request
 def response(resp):
-    result_xpath = '//div[@class="results_links results_links_deep web-result"]'  # noqa
-    url_xpath = './/a[@class="large"]/@href'
-    title_xpath = './/a[@class="large"]//text()'
-    content_xpath = './/div[@class="snippet"]//text()'
     results = []
 
     doc = fromstring(resp.text)
 
+    # parse results
     for r in doc.xpath(result_xpath):
         try:
             res_url = r.xpath(url_xpath)[-1]
         except:
             continue
+
         if not res_url:
             continue
+
         title = html_to_text(''.join(r.xpath(title_xpath)))
         content = html_to_text(''.join(r.xpath(content_xpath)))
+
+        # append result
         results.append({'title': title,
                         'content': content,
                         'url': res_url})
 
+    # return results
     return results
-
-
-#from json import loads
-#search_url = url + 'd.js?{query}&p=1&s={offset}'
-#
-#paging = True
-#
-#
-#def request(query, params):
-#    offset = (params['pageno'] - 1) * 30
-#    q = urlencode({'q': query,
-#                   'l': locale})
-#    params['url'] = search_url.format(query=q, offset=offset)
-#    return params
-#
-#
-#def response(resp):
-#    results = []
-#    search_res = loads(resp.text[resp.text.find('[{'):-2])[:-1]
-#    for r in search_res:
-#        if not r.get('t'):
-#            continue
-#        results.append({'title': r['t'],
-#                       'content': html_to_text(r['a']),
-#                       'url': r['u']})
-#    return results

searx/engines/dummy.py

+## Dummy
+# 
+# @results     empty array
+# @stable      yes
+
+
+# do search-request
 def request(query, params):
     return params
 
 
+# get response from search-request
 def response(resp):
     return []

searx/engines/generalfile.py

+## General Files (Files)
+# 
+# @website     http://www.general-files.org
+# @provide-api no (nothing found)
+# 
+# @using-api   no (because nothing found)
+# @results     HTML (using search portal)
+# @stable      no (HTML can change)
+# @parse       url, title, content
+#
+# @todo        detect torrents?
+
 from lxml import html
 
+# engine dependent config
+categories = ['files']
+paging = True
 
+# search-url
 base_url = 'http://www.general-file.com'
 search_url = base_url + '/files-{letter}/{query}/{pageno}'
 
+# specific xpath variables
 result_xpath = '//table[@class="block-file"]'
 title_xpath = './/h2/a//text()'
 url_xpath = './/h2/a/@href'
 content_xpath = './/p//text()'
 
-paging = True
-
 
+# do search-request
 def request(query, params):
+
     params['url'] = search_url.format(query=query,
                                       letter=query[0],
                                       pageno=params['pageno'])
+
     return params
 
 
+# get response from search-request
 def response(resp):
-
     results = []
+
     dom = html.fromstring(resp.text)
+
+    # parse results
     for result in dom.xpath(result_xpath):
         url = result.xpath(url_xpath)[0]
+
         # skip fast download links
         if not url.startswith('/'):
             continue
+
+        # append result
         results.append({'url': base_url + url,
                         'title': ''.join(result.xpath(title_xpath)),
                         'content': ''.join(result.xpath(content_xpath))})
 
+    # return results
     return results

searx/engines/github.py

+## Github (It)
+# 
+# @website     https://github.com/
+# @provide-api yes (https://developer.github.com/v3/)
+# 
+# @using-api   yes
+# @results     JSON
+# @stable      yes (using api)
+# @parse       url, title, content
+
 from urllib import urlencode
 from json import loads
 from cgi import escape
 
+# engine dependent config
 categories = ['it']
 
+# search-url
 search_url = 'https://api.github.com/search/repositories?sort=stars&order=desc&{query}'  # noqa
 
 accept_header = 'application/vnd.github.preview.text-match+json'
 
 
+# do search-request
 def request(query, params):
     params['url'] = search_url.format(query=urlencode({'q': query}))
+
     params['headers']['Accept'] = accept_header
+
     return params
 
 
+# get response from search-request
 def response(resp):
     results = []
+
     search_res = loads(resp.text)
+
+    # check if items are recieved
     if not 'items' in search_res:
-        return results
+        return []
+
+    # parse results
     for res in search_res['items']:
         title = res['name']
         url = res['html_url']
+
         if res['description']:
             content = escape(res['description'][:500])
         else:
             content = ''
-        results.append({'url': url, 'title': title, 'content': content})
+
+        # append result
+        results.append({'url': url,
+                        'title': title,
+                        'content': content})
+
+    # return results
     return results

searx/engines/piratebay.py

+## Piratebay (Videos, Music, Files)
+# 
+# @website     https://thepiratebay.se
+# @provide-api no (nothing found)
+# 
+# @using-api   no
+# @results     HTML (using search portal)
+# @stable      yes (HTML can change)
+# @parse       url, title, content, seed, leech, magnetlink
+
 from urlparse import urljoin
 from cgi import escape
 from urllib import quote
 from lxml import html
 from operator import itemgetter
 
-categories = ['videos', 'music']
+# engine dependent config
+categories = ['videos', 'music', 'files']
+paging = True
 
+# search-url
 url = 'https://thepiratebay.se/'
 search_url = url + 'search/{search_term}/{pageno}/99/{search_type}'
-search_types = {'videos': '200',
+
+# piratebay specific type-definitions
+search_types = {'files': '0',                
                 'music': '100',
-                'files': '0'}
+                'videos': '200'}
 
+# specific xpath variables
 magnet_xpath = './/a[@title="Download this torrent using magnet"]'
 content_xpath = './/font[@class="detDesc"]//text()'
 
-paging = True
-
 
+# do search-request
 def request(query, params):
-    search_type = search_types.get(params['category'], '200')
+    search_type = search_types.get(params['category'], '0')
+
     params['url'] = search_url.format(search_term=quote(query),
                                       search_type=search_type,
                                       pageno=params['pageno'] - 1)
+
     return params
 
 
+# get response from search-request
 def response(resp):
     results = []
+
     dom = html.fromstring(resp.text)
+
     search_res = dom.xpath('//table[@id="searchResult"]//tr')
 
+    # return empty array if nothing is found
     if not search_res:
-        return results
+        return []
 
+    # parse results
     for result in search_res[1:]:
         link = result.xpath('.//div[@class="detName"]//a')[0]
         href = urljoin(url, link.attrib.get('href'))
         content = escape(' '.join(result.xpath(content_xpath)))
         seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]
 
+        # convert seed to int if possible
         if seed.isdigit():
             seed = int(seed)
         else:
             seed = 0
 
+        # convert leech to int if possible
         if leech.isdigit():
             leech = int(leech)
         else:
             leech = 0
 
         magnetlink = result.xpath(magnet_xpath)[0]
+
+        # append result
         results.append({'url': href,
                         'title': title,
                         'content': content,
                         'magnetlink': magnetlink.attrib['href'],
                         'template': 'torrent.html'})
 
+    # return results sorted by seeder
     return sorted(results, key=itemgetter('seed'), reverse=True)

searx/engines/soundcloud.py

+## Soundcloud (Music)
+# 
+# @website     https://soundcloud.com
+# @provide-api yes (https://developers.soundcloud.com/)
+# 
+# @using-api   yes
+# @results     JSON
+# @stable      yes
+# @parse       url, title, content
+
 from json import loads
 from urllib import urlencode
 
+# engine dependent config
 categories = ['music']
+paging = True
 
+# api-key
 guest_client_id = 'b45b1aa10f1ac2941910a7f0d10f8e28'
-url = 'https://api.soundcloud.com/'
-search_url = url + 'search?{query}&facet=model&limit=20&offset={offset}&linked_partitioning=1&client_id='+guest_client_id  # noqa
 
-paging = True
+# search-url
+url = 'https://api.soundcloud.com/'
+search_url = url + 'search?{query}&facet=model&limit=20&offset={offset}&linked_partitioning=1&client_id={client_id}'
 
 
+# do search-request
 def request(query, params):
     offset = (params['pageno'] - 1) * 20
+
     params['url'] = search_url.format(query=urlencode({'q': query}),
-                                      offset=offset)
+                                      offset=offset,
+                                      client_id=guest_client_id)
+
     return params
 
 
+# get response from search-request
 def response(resp):
     results = []
+
     search_res = loads(resp.text)
+
+    # parse results
     for result in search_res.get('collection', []):
         if result['kind'] in ('track', 'playlist'):
             title = result['title']
             content = result['description']
+
+            # append result
             results.append({'url': result['permalink_url'],
                             'title': title,
                             'content': content})
+
+    # return results
     return results

searx/engines/stackoverflow.py

+## Stackoverflow (It)
+# 
+# @website     https://stackoverflow.com/
+# @provide-api not clear (https://api.stackexchange.com/docs/advanced-search)
+# 
+# @using-api   no
+# @results     HTML
+# @stable      no (HTML can change)
+# @parse       url, title, content
+
 from urlparse import urljoin
 from cgi import escape
 from urllib import urlencode
 from lxml import html
 
+# engine dependent config
 categories = ['it']
+paging = True
 
+# search-url
 url = 'http://stackoverflow.com/'
 search_url = url+'search?{query}&page={pageno}'
-result_xpath = './/div[@class="excerpt"]//text()'
 
-paging = True
+# specific xpath variables
+results_xpath = '//div[contains(@class,"question-summary")]'
+link_xpath = './/div[@class="result-link"]//a|.//div[@class="summary"]//h3//a'
+title_xpath = './/text()'
+content_xpath = './/div[@class="excerpt"]//text()'
 
 
+# do search-request
 def request(query, params):
     params['url'] = search_url.format(query=urlencode({'q': query}),
                                       pageno=params['pageno'])
+
     return params
 
 
+# get response from search-request
 def response(resp):
     results = []
+
     dom = html.fromstring(resp.text)
-    for result in dom.xpath('//div[@class="question-summary search-result"]'):
-        link = result.xpath('.//div[@class="result-link"]//a')[0]
+
+    # parse results
+    for result in dom.xpath(results_xpath):
+        link = result.xpath(link_xpath)[0]
         href = urljoin(url, link.attrib.get('href'))
-        title = escape(' '.join(link.xpath('.//text()')))
-        content = escape(' '.join(result.xpath(result_xpath)))
-        results.append({'url': href, 'title': title, 'content': content})
+        title = escape(' '.join(link.xpath(title_xpath)))
+        content = escape(' '.join(result.xpath(content_xpath)))
+
+        # append result
+        results.append({'url': href, 
+                        'title': title, 
+                        'content': content})
+
+    # return results
     return results

searx/engines/startpage.py

+## Startpage (Web)
+# 
+# @website     https://startpage.com
+# @provide-api no (nothing found)
+# 
+# @using-api   no
+# @results     HTML
+# @stable      no (HTML can change)
+# @parse       url, title, content
+#
+# @todo        paging
+
 from urllib import urlencode
 from lxml import html
 from cgi import escape
+import re
+
+# engine dependent config
+categories = ['general']
+# there is a mechanism to block "bot" search (probably the parameter qid), require storing of qid's between mulitble search-calls
+#paging = False 
+language_support = True
 
-base_url = None
-search_url = None
+# search-url
+base_url = 'https://startpage.com/'
+search_url = base_url + 'do/search'
 
-# TODO paging
-paging = False
-# TODO complete list of country mapping
-country_map = {'en_US': 'eng',
-               'en_UK': 'uk',
-               'nl_NL': 'ned'}
+# specific xpath variables
+# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
+# not ads: div[@class="result"] are the direct childs of div[@id="results"]
+results_xpath = '//div[@class="result"]'
+link_xpath = './/h3/a'
 
 
+# do search-request
 def request(query, params):
+    offset = (params['pageno'] - 1) * 10
     query = urlencode({'q': query})[2:]
+
     params['url'] = search_url
     params['method'] = 'POST'
     params['data'] = {'query': query,
-                      'startat': (params['pageno'] - 1) * 10}  # offset
-    country = country_map.get(params['language'], 'eng')
-    params['cookies']['preferences'] = \
-        'lang_homepageEEEs/air/{country}/N1NsslEEE1N1Nfont_sizeEEEmediumN1Nrecent_results_filterEEE1N1Nlanguage_uiEEEenglishN1Ndisable_open_in_new_windowEEE0N1Ncolor_schemeEEEnewN1Nnum_of_resultsEEE10N1N'.format(country=country)  # noqa
+                      'startat': offset}   
+
+    # set language if specified
+    if params['language'] != 'all':
+        params['data']['with_language'] = 'lang_' + params['language'].split('_')[0]
+
     return params
 
 
+# get response from search-request
 def response(resp):
     results = []
+
     dom = html.fromstring(resp.content)
-    # ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
-    # not ads: div[@class="result"] are the direct childs of div[@id="results"]
-    for result in dom.xpath('//div[@class="result"]'):
-        link = result.xpath('.//h3/a')[0]
+    
+    # parse results
+    for result in dom.xpath(results_xpath):
+        link = result.xpath(link_xpath)[0]
         url = link.attrib.get('href')
-        if url.startswith('http://www.google.')\
-           or url.startswith('https://www.google.'):
-            continue
         title = escape(link.text_content())
 
-        content = ''
+        # block google-ad url's
+        if re.match("^http(s|)://www.google.[a-z]+/aclk.*$", url):
+            continue
+
         if result.xpath('./p[@class="desc"]'):
             content = escape(result.xpath('./p[@class="desc"]')[0].text_content())
+        else:
+            content = ''
 
-        results.append({'url': url, 'title': title, 'content': content})
+        # append result
+        results.append({'url': url, 
+                        'title': title, 
+                        'content': content})
 
+    # return results
     return results

searx/engines/twitter.py

+## Twitter (Social media)
+# 
+# @website     https://www.bing.com/news
+# @provide-api yes (https://dev.twitter.com/docs/using-search)
+# 
+# @using-api   no
+# @results     HTML (using search portal)
+# @stable      no (HTML can change)
+# @parse       url, title, content
+#
+# @todo        publishedDate
+
 from urlparse import urljoin
 from urllib import urlencode
 from lxml import html
 from cgi import escape
 
+# engine dependent config
 categories = ['social media']
+language_support = True
 
+# search-url
 base_url = 'https://twitter.com/'
 search_url = base_url+'search?'
+
+# specific xpath variables
+results_xpath = '//li[@data-item-type="tweet"]'
+link_xpath = './/small[@class="time"]//a'
 title_xpath = './/span[@class="username js-action-profile-name"]//text()'
 content_xpath = './/p[@class="js-tweet-text tweet-text"]//text()'
 
 
+# do search-request
 def request(query, params):
     params['url'] = search_url + urlencode({'q': query})
+
+    # set language if specified
+    if params['language'] != 'all':
+        params['cookies']['lang'] = params['language'].split('_')[0]
+
     return params
 
 
+# get response from search-request
 def response(resp):
     results = []
+
     dom = html.fromstring(resp.text)
-    for tweet in dom.xpath('//li[@data-item-type="tweet"]'):
-        link = tweet.xpath('.//small[@class="time"]//a')[0]
+
+    # parse results
+    for tweet in dom.xpath(results_xpath):
+        link = tweet.xpath(link_xpath)[0]
         url = urljoin(base_url, link.attrib.get('href'))
         title = ''.join(tweet.xpath(title_xpath))
         content = escape(''.join(tweet.xpath(content_xpath)))
+
+        # append result
         results.append({'url': url,
                         'title': title,
                         'content': content})
+
+    # return results
     return results

searx/engines/wikipedia.py

+## Wikipedia (Web)
+# 
+# @website     http://www.wikipedia.org
+# @provide-api yes (http://www.mediawiki.org/wiki/API:Search)
+# 
+# @using-api   yes
+# @results     JSON
+# @stable      yes
+# @parse       url, title 
+#
+# @todo        content
+
 from json import loads
 from urllib import urlencode, quote
 
-url = 'https://{language}.wikipedia.org/'
-
-search_url = url + 'w/api.php?action=query&list=search&{query}&srprop=timestamp&format=json&sroffset={offset}'  # noqa
-
-number_of_results = 10
-
+# engine dependent config
+categories = ['general']
 language_support = True
+paging = True
+number_of_results = 1
+    
+# search-url
+url = 'https://{language}.wikipedia.org/'
+search_url = url + 'w/api.php?action=query&list=search&{query}&srprop=timestamp&format=json&sroffset={offset}&srlimit={limit}'  # noqa
 
 
+# do search-request
 def request(query, params):
-    offset = (params['pageno'] - 1) * 10
+    offset = (params['pageno'] - 1) * number_of_results
+
     if params['language'] == 'all':
         language = 'en'
     else:
         language = params['language'].split('_')[0]
+    
+    # write search-language back to params, required in response
     params['language'] = language
+
     params['url'] = search_url.format(query=urlencode({'srsearch': query}),
                                       offset=offset,
+                                      limit=number_of_results,
                                       language=language)
+
     return params
 
 
+# get response from search-request
 def response(resp):
+    results = []
+
     search_results = loads(resp.text)
-    res = search_results.get('query', {}).get('search', [])
-    return [{'url': url.format(language=resp.search_params['language']) + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8')),  # noqa
-        'title': result['title']} for result in res[:int(number_of_results)]]
+
+    # return empty array if there are no results
+    if not search_results.get('query', {}).get('search'):
+        return []
+
+    # parse results
+    for result in search_results['query']['search']:
+        res_url = url.format(language=resp.search_params['language']) + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8'))
+        
+        # append result
+        results.append({'url': res_url,
+                        'title': result['title'],
+                        'content': ''})
+
+    # return results
+    return results

searx/engines/youtube.py

+## Youtube (Videos)
+# 
+# @website     https://www.youtube.com/
+# @provide-api yes (http://gdata-samples-youtube-search-py.appspot.com/)
+# 
+# @using-api   yes
+# @results     JSON
+# @stable      yes
+# @parse       url, title, content, publishedDate, thumbnail
+
 from json import loads
 from urllib import urlencode
 from dateutil import parser
 
+# engine dependent config
 categories = ['videos']
-
-search_url = ('https://gdata.youtube.com/feeds/api/videos'
-              '?alt=json&{query}&start-index={index}&max-results=25')  # noqa
-
 paging = True
+language_support = True
+
+# search-url
+base_url = 'https://gdata.youtube.com/feeds/api/videos'
+search_url = base_url + '?alt=json&{query}&start-index={index}&max-results=5'  # noqa
 
 
+# do search-request
 def request(query, params):
-    index = (params['pageno'] - 1) * 25 + 1
+    index = (params['pageno'] - 1) * 5 + 1
+
     params['url'] = search_url.format(query=urlencode({'q': query}),
                                       index=index)
+
+    # add language tag if specified
+    if params['language'] != 'all':
+        params['url'] += '&lr=' + params['language'].split('_')[0]
+
     return params
 
 
+# get response from search-request
 def response(resp):
     results = []
+
     search_results = loads(resp.text)
+
+    # return empty array if there are no results
     if not 'feed' in search_results:
-        return results
+        return []
+
     feed = search_results['feed']
 
+    # parse results
     for result in feed['entry']:
         url = [x['href'] for x in result['link'] if x['type'] == 'text/html']
+
         if not url:
             return
+
         # remove tracking
         url = url[0].replace('feature=youtube_gdata', '')
         if url.endswith('&'):
             url = url[:-1]
+
         title = result['title']['$t']
         content = ''
         thumbnail = ''
 
-#"2013-12-31T15:22:51.000Z"
         pubdate = result['published']['$t']
         publishedDate = parser.parse(pubdate)
 
         else:
             content = result['content']['$t']
 
+        # append result
         results.append({'url': url,
                         'title': title,
                         'content': content,
                         'publishedDate': publishedDate,
                         'thumbnail': thumbnail})
 
+    # return results
     return results

searx/settings.yml

 engines:
   - name : wikipedia
     engine : wikipedia
-    number_of_results : 1
-    paging : False
     shortcut : wp
+#    number_of_results : 1 # default is 1
 
   - name : bing
     engine : bing
 
   - name : deviantart
     engine : deviantart
-    categories : images
     shortcut : da
     timeout: 3.0
 
 
   - name : duckduckgo
     engine : duckduckgo
-    locale : en-us
     shortcut : ddg
 
 # down - website is under criminal investigation by the UK
 
   - name : general-file
     engine : generalfile
-    categories : files
     shortcut : gf
 
   - name : github
     engine : github
-    categories : it
     shortcut : gh
 
   - name : google
 
   - name : piratebay
     engine : piratebay
-    categories : videos, music, files
     shortcut : tpb
 
   - name : soundcloud
     engine : soundcloud
-    categories : music
     shortcut : sc
 
   - name : stackoverflow
     engine : stackoverflow
-    categories : it
     shortcut : st
 
   - name : startpage
     engine : startpage
-    base_url : 'https://startpage.com/'
-    search_url : 'https://startpage.com/do/search'
     shortcut : sp
 
 # +30% page load time
 
   - name : twitter
     engine : twitter
-    categories : social media
     shortcut : tw
 
 # maybe in a fun category
 
   - name : youtube
     engine : youtube
-    categories : videos
     shortcut : yt
 
   - name : dailymotion
     engine : dailymotion
-    locale : en_US
-    categories : videos
     shortcut : dm
 
   - name : vimeo