123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835
  1. #!/usr/bin/env python
  2. '''
  3. searx is free software: you can redistribute it and/or modify
  4. it under the terms of the GNU Affero General Public License as published by
  5. the Free Software Foundation, either version 3 of the License, or
  6. (at your option) any later version.
  7. searx is distributed in the hope that it will be useful,
  8. but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. GNU Affero General Public License for more details.
  11. You should have received a copy of the GNU Affero General Public License
  12. along with searx. If not, see < http://www.gnu.org/licenses/ >.
  13. (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
  14. '''
  15. if __name__ == '__main__':
  16. from sys import path
  17. from os.path import realpath, dirname
  18. path.append(realpath(dirname(realpath(__file__)) + '/../'))
  19. import cStringIO
  20. import hashlib
  21. import hmac
  22. import json
  23. import os
  24. import requests
  25. from searx import logger
  26. logger = logger.getChild('webapp')
  27. try:
  28. from pygments import highlight
  29. from pygments.lexers import get_lexer_by_name
  30. from pygments.formatters import HtmlFormatter
  31. except:
  32. logger.critical("cannot import dependency: pygments")
  33. from sys import exit
  34. exit(1)
  35. from cgi import escape
  36. from datetime import datetime, timedelta
  37. from urllib import urlencode
  38. from urlparse import urlparse, urljoin
  39. from werkzeug.contrib.fixers import ProxyFix
  40. from flask import (
  41. Flask, request, render_template, url_for, Response, make_response,
  42. redirect, send_from_directory
  43. )
  44. from flask_babel import Babel, gettext, format_date, format_decimal
  45. from flask.json import jsonify
  46. from searx import settings, searx_dir, searx_debug
  47. from searx.engines import (
  48. categories, engines, engine_shortcuts, get_engines_stats, initialize_engines
  49. )
  50. from searx.utils import (
  51. UnicodeWriter, highlight_content, html_to_text, get_themes,
  52. get_static_files, get_result_templates, gen_useragent, dict_subset,
  53. prettify_url
  54. )
  55. from searx.version import VERSION_STRING
  56. from searx.languages import language_codes
  57. from searx.search import SearchWithPlugins, get_search_query_from_webapp
  58. from searx.query import RawTextQuery
  59. from searx.autocomplete import searx_bang, backends as autocomplete_backends
  60. from searx.plugins import plugins
  61. from searx.preferences import Preferences, ValidationException
  62. from searx.answerers import answerers
  63. # check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed.
  64. # They are needed for SSL connection without trouble, see #298
  65. try:
  66. import OpenSSL.SSL # NOQA
  67. import ndg.httpsclient # NOQA
  68. import pyasn1 # NOQA
  69. except ImportError:
  70. logger.critical("The pyopenssl, ndg-httpsclient, pyasn1 packages have to be installed.\n"
  71. "Some HTTPS connections will fail")
  72. # serve pages with HTTP/1.1
  73. from werkzeug.serving import WSGIRequestHandler
  74. WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
  75. static_path, templates_path, themes =\
  76. get_themes(settings['ui']['themes_path']
  77. if settings['ui']['themes_path']
  78. else searx_dir)
  79. default_theme = settings['ui']['default_theme']
  80. static_files = get_static_files(searx_dir)
  81. result_templates = get_result_templates(searx_dir)
  82. app = Flask(
  83. __name__,
  84. static_folder=static_path,
  85. template_folder=templates_path
  86. )
  87. app.jinja_env.trim_blocks = True
  88. app.jinja_env.lstrip_blocks = True
  89. app.secret_key = settings['server']['secret_key']
  90. if not searx_debug or os.environ.get("WERKZEUG_RUN_MAIN") == "true":
  91. initialize_engines(settings['engines'])
  92. babel = Babel(app)
  93. rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'glk', 'he',
  94. 'ku', 'mzn', 'pnb'', ''ps', 'sd', 'ug', 'ur', 'yi']
  95. global_favicons = []
  96. for indice, theme in enumerate(themes):
  97. global_favicons.append([])
  98. theme_img_path = searx_dir + "/static/themes/" + theme + "/img/icons/"
  99. for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
  100. global_favicons[indice].extend(filenames)
  101. # used when translating category names
  102. _category_names = (gettext('files'),
  103. gettext('general'),
  104. gettext('music'),
  105. gettext('social media'),
  106. gettext('images'),
  107. gettext('videos'),
  108. gettext('it'),
  109. gettext('news'),
  110. gettext('map'),
  111. gettext('science'))
  112. outgoing_proxies = settings['outgoing'].get('proxies', None)
  113. @babel.localeselector
  114. def get_locale():
  115. locale = request.accept_languages.best_match(settings['locales'].keys())
  116. if request.preferences.get_value('locale') != '':
  117. locale = request.preferences.get_value('locale')
  118. if 'locale' in request.args\
  119. and request.args['locale'] in settings['locales']:
  120. locale = request.args['locale']
  121. if 'locale' in request.form\
  122. and request.form['locale'] in settings['locales']:
  123. locale = request.form['locale']
  124. return locale
  125. # code-highlighter
  126. @app.template_filter('code_highlighter')
  127. def code_highlighter(codelines, language=None):
  128. if not language:
  129. language = 'text'
  130. try:
  131. # find lexer by programing language
  132. lexer = get_lexer_by_name(language, stripall=True)
  133. except:
  134. # if lexer is not found, using default one
  135. logger.debug('highlighter cannot find lexer for {0}'.format(language))
  136. lexer = get_lexer_by_name('text', stripall=True)
  137. html_code = ''
  138. tmp_code = ''
  139. last_line = None
  140. # parse lines
  141. for line, code in codelines:
  142. if not last_line:
  143. line_code_start = line
  144. # new codeblock is detected
  145. if last_line is not None and\
  146. last_line + 1 != line:
  147. # highlight last codepart
  148. formatter = HtmlFormatter(linenos='inline',
  149. linenostart=line_code_start)
  150. html_code = html_code + highlight(tmp_code, lexer, formatter)
  151. # reset conditions for next codepart
  152. tmp_code = ''
  153. line_code_start = line
  154. # add codepart
  155. tmp_code += code + '\n'
  156. # update line
  157. last_line = line
  158. # highlight last codepart
  159. formatter = HtmlFormatter(linenos='inline', linenostart=line_code_start)
  160. html_code = html_code + highlight(tmp_code, lexer, formatter)
  161. return html_code
  162. # Extract domain from url
  163. @app.template_filter('extract_domain')
  164. def extract_domain(url):
  165. return urlparse(url)[1]
  166. def get_base_url():
  167. if settings['server']['base_url']:
  168. hostname = settings['server']['base_url']
  169. else:
  170. scheme = 'http'
  171. if request.is_secure:
  172. scheme = 'https'
  173. hostname = url_for('index', _external=True, _scheme=scheme)
  174. return hostname
  175. def get_current_theme_name(override=None):
  176. """Returns theme name.
  177. Checks in this order:
  178. 1. override
  179. 2. cookies
  180. 3. settings"""
  181. if override and override in themes:
  182. return override
  183. theme_name = request.args.get('theme', request.preferences.get_value('theme'))
  184. if theme_name not in themes:
  185. theme_name = default_theme
  186. return theme_name
  187. def get_result_template(theme, template_name):
  188. themed_path = theme + '/result_templates/' + template_name
  189. if themed_path in result_templates:
  190. return themed_path
  191. return 'result_templates/' + template_name
  192. def url_for_theme(endpoint, override_theme=None, **values):
  193. if endpoint == 'static' and values.get('filename'):
  194. theme_name = get_current_theme_name(override=override_theme)
  195. filename_with_theme = "themes/{}/{}".format(theme_name, values['filename'])
  196. if filename_with_theme in static_files:
  197. values['filename'] = filename_with_theme
  198. return url_for(endpoint, **values)
  199. def proxify(url):
  200. if url.startswith('//'):
  201. url = 'https:' + url
  202. if not settings.get('result_proxy'):
  203. return url
  204. url_params = dict(mortyurl=url.encode('utf-8'))
  205. if settings['result_proxy'].get('key'):
  206. url_params['mortyhash'] = hmac.new(settings['result_proxy']['key'],
  207. url.encode('utf-8'),
  208. hashlib.sha256).hexdigest()
  209. return '{0}?{1}'.format(settings['result_proxy']['url'],
  210. urlencode(url_params))
  211. def image_proxify(url):
  212. if url.startswith('//'):
  213. url = 'https:' + url
  214. if not request.preferences.get_value('image_proxy'):
  215. return url
  216. if settings.get('result_proxy'):
  217. return proxify(url)
  218. h = hmac.new(settings['server']['secret_key'], url.encode('utf-8'), hashlib.sha256).hexdigest()
  219. return '{0}?{1}'.format(url_for('image_proxy'),
  220. urlencode(dict(url=url.encode('utf-8'), h=h)))
  221. def render(template_name, override_theme=None, **kwargs):
  222. disabled_engines = request.preferences.engines.get_disabled()
  223. enabled_categories = set(category for engine_name in engines
  224. for category in engines[engine_name].categories
  225. if (engine_name, category) not in disabled_engines)
  226. if 'categories' not in kwargs:
  227. kwargs['categories'] = ['general']
  228. kwargs['categories'].extend(x for x in
  229. sorted(categories.keys())
  230. if x != 'general'
  231. and x in enabled_categories)
  232. if 'all_categories' not in kwargs:
  233. kwargs['all_categories'] = ['general']
  234. kwargs['all_categories'].extend(x for x in
  235. sorted(categories.keys())
  236. if x != 'general')
  237. if 'selected_categories' not in kwargs:
  238. kwargs['selected_categories'] = []
  239. for arg in request.args:
  240. if arg.startswith('category_'):
  241. c = arg.split('_', 1)[1]
  242. if c in categories:
  243. kwargs['selected_categories'].append(c)
  244. if not kwargs['selected_categories']:
  245. cookie_categories = request.preferences.get_value('categories')
  246. for ccateg in cookie_categories:
  247. kwargs['selected_categories'].append(ccateg)
  248. if not kwargs['selected_categories']:
  249. kwargs['selected_categories'] = ['general']
  250. if 'autocomplete' not in kwargs:
  251. kwargs['autocomplete'] = request.preferences.get_value('autocomplete')
  252. if get_locale() in rtl_locales and 'rtl' not in kwargs:
  253. kwargs['rtl'] = True
  254. kwargs['searx_version'] = VERSION_STRING
  255. kwargs['method'] = request.preferences.get_value('method')
  256. kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
  257. kwargs['language_codes'] = language_codes
  258. if 'current_language' not in kwargs:
  259. kwargs['current_language'] = request.preferences.get_value('language')
  260. # override url_for function in templates
  261. kwargs['url_for'] = url_for_theme
  262. kwargs['image_proxify'] = image_proxify
  263. kwargs['proxify'] = proxify if settings.get('result_proxy') else None
  264. kwargs['get_result_template'] = get_result_template
  265. kwargs['theme'] = get_current_theme_name(override=override_theme)
  266. kwargs['template_name'] = template_name
  267. kwargs['cookies'] = request.cookies
  268. kwargs['errors'] = request.errors
  269. kwargs['instance_name'] = settings['general']['instance_name']
  270. kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')
  271. kwargs['scripts'] = set()
  272. for plugin in request.user_plugins:
  273. for script in plugin.js_dependencies:
  274. kwargs['scripts'].add(script)
  275. kwargs['styles'] = set()
  276. for plugin in request.user_plugins:
  277. for css in plugin.css_dependencies:
  278. kwargs['styles'].add(css)
  279. return render_template(
  280. '{}/{}'.format(kwargs['theme'], template_name), **kwargs)
  281. @app.before_request
  282. def pre_request():
  283. request.errors = []
  284. preferences = Preferences(themes, categories.keys(), engines, plugins)
  285. request.preferences = preferences
  286. try:
  287. preferences.parse_cookies(request.cookies)
  288. except:
  289. request.errors.append(gettext('Invalid settings, please edit your preferences'))
  290. # merge GET, POST vars
  291. # request.form
  292. request.form = dict(request.form.items())
  293. for k, v in request.args.items():
  294. if k not in request.form:
  295. request.form[k] = v
  296. # request.user_plugins
  297. request.user_plugins = []
  298. allowed_plugins = preferences.plugins.get_enabled()
  299. disabled_plugins = preferences.plugins.get_disabled()
  300. for plugin in plugins:
  301. if ((plugin.default_on and plugin.id not in disabled_plugins)
  302. or plugin.id in allowed_plugins):
  303. request.user_plugins.append(plugin)
  304. @app.route('/search', methods=['GET', 'POST'])
  305. @app.route('/', methods=['GET', 'POST'])
  306. def index():
  307. """Render index page.
  308. Supported outputs: html, json, csv, rss.
  309. """
  310. if request.form.get('q') is None:
  311. return render(
  312. 'index.html',
  313. )
  314. # search
  315. search_query = None
  316. result_container = None
  317. try:
  318. search_query = get_search_query_from_webapp(request.preferences, request.form)
  319. # search = Search(search_query) # without plugins
  320. search = SearchWithPlugins(search_query, request.user_plugins, request)
  321. result_container = search.search()
  322. except:
  323. request.errors.append(gettext('search error'))
  324. logger.exception('search error')
  325. return render(
  326. 'index.html',
  327. )
  328. results = result_container.get_ordered_results()
  329. # UI
  330. advanced_search = request.form.get('advanced_search', None)
  331. output_format = request.form.get('format', 'html')
  332. if output_format not in ['html', 'csv', 'json', 'rss']:
  333. output_format = 'html'
  334. # output
  335. for result in results:
  336. if output_format == 'html':
  337. if 'content' in result and result['content']:
  338. result['content'] = highlight_content(escape(result['content'][:1024]),
  339. search_query.query.encode('utf-8'))
  340. result['title'] = highlight_content(escape(result['title'] or u''),
  341. search_query.query.encode('utf-8'))
  342. else:
  343. if result.get('content'):
  344. result['content'] = html_to_text(result['content']).strip()
  345. # removing html content and whitespace duplications
  346. result['title'] = ' '.join(html_to_text(result['title']).strip().split())
  347. result['pretty_url'] = prettify_url(result['url'])
  348. # TODO, check if timezone is calculated right
  349. if 'publishedDate' in result:
  350. try: # test if publishedDate >= 1900 (datetime module bug)
  351. result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S%z')
  352. except ValueError:
  353. result['publishedDate'] = None
  354. else:
  355. if result['publishedDate'].replace(tzinfo=None) >= datetime.now() - timedelta(days=1):
  356. timedifference = datetime.now() - result['publishedDate'].replace(tzinfo=None)
  357. minutes = int((timedifference.seconds / 60) % 60)
  358. hours = int(timedifference.seconds / 60 / 60)
  359. if hours == 0:
  360. result['publishedDate'] = gettext(u'{minutes} minute(s) ago').format(minutes=minutes)
  361. else:
  362. result['publishedDate'] = gettext(u'{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes) # noqa
  363. else:
  364. result['publishedDate'] = format_date(result['publishedDate'])
  365. number_of_results = result_container.results_number()
  366. if number_of_results < result_container.results_length():
  367. number_of_results = 0
  368. if output_format == 'json':
  369. return Response(json.dumps({'query': search_query.query,
  370. 'number_of_results': number_of_results,
  371. 'results': results,
  372. 'answers': list(result_container.answers),
  373. 'infoboxes': result_container.infoboxes,
  374. 'suggestions': list(result_container.suggestions)}),
  375. mimetype='application/json')
  376. elif output_format == 'csv':
  377. csv = UnicodeWriter(cStringIO.StringIO())
  378. keys = ('title', 'url', 'content', 'host', 'engine', 'score')
  379. csv.writerow(keys)
  380. for row in results:
  381. row['host'] = row['parsed_url'].netloc
  382. csv.writerow([row.get(key, '') for key in keys])
  383. csv.stream.seek(0)
  384. response = Response(csv.stream.read(), mimetype='application/csv')
  385. cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query.encode('utf-8'))
  386. response.headers.add('Content-Disposition', cont_disp)
  387. return response
  388. elif output_format == 'rss':
  389. response_rss = render(
  390. 'opensearch_response_rss.xml',
  391. results=results,
  392. q=request.form['q'],
  393. number_of_results=number_of_results,
  394. base_url=get_base_url()
  395. )
  396. return Response(response_rss, mimetype='text/xml')
  397. return render(
  398. 'results.html',
  399. results=results,
  400. q=request.form['q'],
  401. selected_categories=search_query.categories,
  402. pageno=search_query.pageno,
  403. time_range=search_query.time_range,
  404. number_of_results=format_decimal(number_of_results),
  405. advanced_search=advanced_search,
  406. suggestions=result_container.suggestions,
  407. answers=result_container.answers,
  408. infoboxes=result_container.infoboxes,
  409. paging=result_container.paging,
  410. current_language=search_query.lang,
  411. base_url=get_base_url(),
  412. theme=get_current_theme_name(),
  413. favicons=global_favicons[themes.index(get_current_theme_name())]
  414. )
  415. @app.route('/about', methods=['GET'])
  416. def about():
  417. """Render about page"""
  418. return render(
  419. 'about.html',
  420. )
  421. @app.route('/autocompleter', methods=['GET', 'POST'])
  422. def autocompleter():
  423. """Return autocompleter results"""
  424. # set blocked engines
  425. disabled_engines = request.preferences.engines.get_disabled()
  426. # parse query
  427. raw_text_query = RawTextQuery(request.form.get('q', '').encode('utf-8'), disabled_engines)
  428. raw_text_query.parse_query()
  429. # check if search query is set
  430. if not raw_text_query.getSearchQuery():
  431. return '', 400
  432. # run autocompleter
  433. completer = autocomplete_backends.get(request.preferences.get_value('autocomplete'))
  434. # parse searx specific autocompleter results like !bang
  435. raw_results = searx_bang(raw_text_query)
  436. # normal autocompletion results only appear if max 3 inner results returned
  437. if len(raw_results) <= 3 and completer:
  438. # get language from cookie
  439. language = request.preferences.get_value('language')
  440. if not language or language == 'all':
  441. language = 'en'
  442. else:
  443. language = language.split('-')[0]
  444. # run autocompletion
  445. raw_results.extend(completer(raw_text_query.getSearchQuery(), language))
  446. # parse results (write :language and !engine back to result string)
  447. results = []
  448. for result in raw_results:
  449. raw_text_query.changeSearchQuery(result)
  450. # add parsed result
  451. results.append(raw_text_query.getFullQuery())
  452. # return autocompleter results
  453. if request.form.get('format') == 'x-suggestions':
  454. return Response(json.dumps([raw_text_query.query, results]),
  455. mimetype='application/json')
  456. return Response(json.dumps(results),
  457. mimetype='application/json')
  458. @app.route('/preferences', methods=['GET', 'POST'])
  459. def preferences():
  460. """Render preferences page && save user preferences"""
  461. # save preferences
  462. if request.method == 'POST':
  463. resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
  464. try:
  465. request.preferences.parse_form(request.form)
  466. except ValidationException:
  467. request.errors.append(gettext('Invalid settings, please edit your preferences'))
  468. return resp
  469. return request.preferences.save(resp)
  470. # render preferences
  471. image_proxy = request.preferences.get_value('image_proxy')
  472. lang = request.preferences.get_value('language')
  473. disabled_engines = request.preferences.engines.get_disabled()
  474. allowed_plugins = request.preferences.plugins.get_enabled()
  475. # stats for preferences page
  476. stats = {}
  477. for c in categories:
  478. for e in categories[c]:
  479. stats[e.name] = {'time': None,
  480. 'warn_timeout': False,
  481. 'warn_time': False}
  482. if e.timeout > settings['outgoing']['request_timeout']:
  483. stats[e.name]['warn_timeout'] = True
  484. # get first element [0], the engine time,
  485. # and then the second element [1] : the time (the first one is the label)
  486. for engine_stat in get_engines_stats()[0][1]:
  487. stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)
  488. if engine_stat.get('avg') > settings['outgoing']['request_timeout']:
  489. stats[engine_stat.get('name')]['warn_time'] = True
  490. # end of stats
  491. return render('preferences.html',
  492. locales=settings['locales'],
  493. current_locale=get_locale(),
  494. image_proxy=image_proxy,
  495. engines_by_category=categories,
  496. stats=stats,
  497. answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
  498. disabled_engines=disabled_engines,
  499. autocomplete_backends=autocomplete_backends,
  500. shortcuts={y: x for x, y in engine_shortcuts.items()},
  501. themes=themes,
  502. plugins=plugins,
  503. allowed_plugins=allowed_plugins,
  504. theme=get_current_theme_name(),
  505. preferences=True)
  506. @app.route('/image_proxy', methods=['GET'])
  507. def image_proxy():
  508. url = request.args.get('url').encode('utf-8')
  509. if not url:
  510. return '', 400
  511. h = hmac.new(settings['server']['secret_key'], url, hashlib.sha256).hexdigest()
  512. if h != request.args.get('h'):
  513. return '', 400
  514. headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})
  515. headers['User-Agent'] = gen_useragent()
  516. resp = requests.get(url,
  517. stream=True,
  518. timeout=settings['outgoing']['request_timeout'],
  519. headers=headers,
  520. proxies=outgoing_proxies)
  521. if resp.status_code == 304:
  522. return '', resp.status_code
  523. if resp.status_code != 200:
  524. logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))
  525. if resp.status_code >= 400:
  526. return '', resp.status_code
  527. return '', 400
  528. if not resp.headers.get('content-type', '').startswith('image/'):
  529. logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))
  530. return '', 400
  531. img = ''
  532. chunk_counter = 0
  533. for chunk in resp.iter_content(1024 * 1024):
  534. chunk_counter += 1
  535. if chunk_counter > 5:
  536. return '', 502 # Bad gateway - file is too big (>5M)
  537. img += chunk
  538. headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})
  539. return Response(img, mimetype=resp.headers['content-type'], headers=headers)
  540. @app.route('/stats', methods=['GET'])
  541. def stats():
  542. """Render engine statistics page."""
  543. stats = get_engines_stats()
  544. return render(
  545. 'stats.html',
  546. stats=stats,
  547. )
  548. @app.route('/robots.txt', methods=['GET'])
  549. def robots():
  550. return Response("""User-agent: *
  551. Allow: /
  552. Allow: /about
  553. Disallow: /stats
  554. Disallow: /preferences
  555. Disallow: /*?*q=*
  556. """, mimetype='text/plain')
  557. @app.route('/opensearch.xml', methods=['GET'])
  558. def opensearch():
  559. method = 'post'
  560. if request.preferences.get_value('method') == 'GET':
  561. method = 'get'
  562. # chrome/chromium only supports HTTP GET....
  563. if request.headers.get('User-Agent', '').lower().find('webkit') >= 0:
  564. method = 'get'
  565. ret = render('opensearch.xml',
  566. opensearch_method=method,
  567. host=get_base_url(),
  568. urljoin=urljoin)
  569. resp = Response(response=ret,
  570. status=200,
  571. mimetype="text/xml")
  572. return resp
  573. @app.route('/favicon.ico')
  574. def favicon():
  575. return send_from_directory(os.path.join(app.root_path,
  576. 'static/themes',
  577. get_current_theme_name(),
  578. 'img'),
  579. 'favicon.png',
  580. mimetype='image/vnd.microsoft.icon')
  581. @app.route('/clear_cookies')
  582. def clear_cookies():
  583. resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
  584. for cookie_name in request.cookies:
  585. resp.delete_cookie(cookie_name)
  586. return resp
  587. @app.route('/config')
  588. def config():
  589. return jsonify({'categories': categories.keys(),
  590. 'engines': [{'name': engine_name,
  591. 'categories': engine.categories,
  592. 'shortcut': engine.shortcut,
  593. 'enabled': not engine.disabled,
  594. 'paging': engine.paging,
  595. 'language_support': engine.language_support,
  596. 'supported_languages':
  597. engine.supported_languages.keys()
  598. if isinstance(engine.supported_languages, dict)
  599. else engine.supported_languages,
  600. 'safesearch': engine.safesearch,
  601. 'time_range_support': engine.time_range_support,
  602. 'timeout': engine.timeout}
  603. for engine_name, engine in engines.items()],
  604. 'plugins': [{'name': plugin.name,
  605. 'enabled': plugin.default_on}
  606. for plugin in plugins],
  607. 'instance_name': settings['general']['instance_name'],
  608. 'locales': settings['locales'],
  609. 'default_locale': settings['ui']['default_locale'],
  610. 'autocomplete': settings['search']['autocomplete'],
  611. 'safe_search': settings['search']['safe_search'],
  612. 'default_theme': settings['ui']['default_theme'],
  613. 'version': VERSION_STRING})
  614. @app.errorhandler(404)
  615. def page_not_found(e):
  616. return render('404.html'), 404
  617. def run():
  618. app.run(
  619. debug=searx_debug,
  620. use_debugger=searx_debug,
  621. port=settings['server']['port'],
  622. host=settings['server']['bind_address'],
  623. threaded=True
  624. )
  625. class ReverseProxyPathFix(object):
  626. '''Wrap the application in this middleware and configure the
  627. front-end server to add these headers, to let you quietly bind
  628. this to a URL other than / and to an HTTP scheme that is
  629. different than what is used locally.
  630. http://flask.pocoo.org/snippets/35/
  631. In nginx:
  632. location /myprefix {
  633. proxy_pass http://127.0.0.1:8000;
  634. proxy_set_header Host $host;
  635. proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  636. proxy_set_header X-Scheme $scheme;
  637. proxy_set_header X-Script-Name /myprefix;
  638. }
  639. :param app: the WSGI application
  640. '''
  641. def __init__(self, app):
  642. self.app = app
  643. def __call__(self, environ, start_response):
  644. script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
  645. if script_name:
  646. environ['SCRIPT_NAME'] = script_name
  647. path_info = environ['PATH_INFO']
  648. if path_info.startswith(script_name):
  649. environ['PATH_INFO'] = path_info[len(script_name):]
  650. scheme = environ.get('HTTP_X_SCHEME', '')
  651. if scheme:
  652. environ['wsgi.url_scheme'] = scheme
  653. return self.app(environ, start_response)
  654. application = app
  655. # patch app to handle non root url-s behind proxy & wsgi
  656. app.wsgi_app = ReverseProxyPathFix(ProxyFix(application.wsgi_app))
  657. if __name__ == "__main__":
  658. run()