searx/searx/webapp.py

#!/usr/bin/env python

'''
searx is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

searx is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with searx. If not, see < http://www.gnu.org/licenses/ >.

(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''

import json
import cStringIO
import os

from flask import (
    Flask, request, render_template, url_for, Response, make_response,
    redirect, send_from_directory
)
from flask.ext.babel import Babel
from searx import settings, searx_dir
from searx.engines import (
    search, categories, engines, get_engines_stats, engine_shortcuts
)
from searx.utils import UnicodeWriter, highlight_content, html_to_text
from searx.languages import language_codes


app = Flask(
    __name__,
    static_folder=os.path.join(searx_dir, 'static'),
    template_folder=os.path.join(searx_dir, 'templates')
)

app.secret_key = settings['server']['secret_key']

babel = Babel(app)

#TODO configurable via settings.yml
favicons = ['wikipedia', 'youtube', 'vimeo', 'soundcloud',
            'twitter', 'stackoverflow', 'github']


@babel.localeselector
def get_locale():
    locale = request.accept_languages.best_match(settings['locales'].keys())

    if request.cookies.get('locale', '') in settings['locales']:
        locale = request.cookies.get('locale', '')

    if 'locale' in request.args\
       and request.args['locale'] in settings['locales']:
        locale = request.args['locale']

    if 'locale' in request.form\
       and request.form['locale'] in settings['locales']:
        locale = request.form['locale']

    return locale


def get_base_url():
    if settings['server']['base_url']:
        hostname = settings['server']['base_url']
    else:
        scheme = 'http'
        if request.is_secure:
            scheme = 'https'
        hostname = url_for('index', _external=True, _scheme=scheme)
    return hostname


def render(template_name, **kwargs):
    global categories
    kwargs['categories'] = ['general']
    kwargs['categories'].extend(x for x in
                                sorted(categories.keys()) if x != 'general')
    if not 'selected_categories' in kwargs:
        kwargs['selected_categories'] = []
        cookie_categories = request.cookies.get('categories', '').split(',')
        for ccateg in cookie_categories:
            if ccateg in categories:
                kwargs['selected_categories'].append(ccateg)
        if not len(kwargs['selected_categories']):
            kwargs['selected_categories'] = ['general']
    return render_template(template_name, **kwargs)


def parse_query(query):
    query_engines = []
    query_parts = query.split()

    if query_parts[0].startswith('!'):
        prefix = query_parts[0][1:].replace('_', ' ')
        if prefix in engine_shortcuts:
            query_engines.append({'category': 'none',
                                  'name': engine_shortcuts[prefix]})
        elif prefix in engines:
            query_engines.append({'category': 'none',
                                  'name': prefix})
        elif prefix in categories:
            query_engines.extend({'category': prefix,
                                  'name': engine.name}
                                 for engine in categories[prefix])

    if len(query_engines):
        query = query.replace(query_parts[0], '', 1).strip()
    return query, query_engines


@app.route('/', methods=['GET', 'POST'])
def index():
    """Render index page.

    Supported outputs: html, json, csv, rss.
    """
    paging = False
    lang = 'all'

    if request.cookies.get('language')\
       and request.cookies['language'] in (x[0] for x in language_codes):
        lang = request.cookies['language']

    if request.method == 'POST':
        request_data = request.form
    else:
        request_data = request.args
    if not request_data.get('q'):
        return render('index.html')

    pageno_param = request_data.get('pageno', '1')
    if not pageno_param.isdigit() or int(pageno_param) < 1:
        return render('index.html')

    pageno = int(pageno_param)

    selected_categories = []

    query, selected_engines = parse_query(request_data['q'].encode('utf-8'))

    if len(selected_engines):
        selected_categories = list(set(engine['category']
                                       for engine in selected_engines))
    else:
        for pd_name, pd in request_data.items():
            if pd_name.startswith('category_'):
                category = pd_name[9:]
                if not category in categories:
                    continue
                selected_categories.append(category)
        if not len(selected_categories):
            cookie_categories = request.cookies.get('categories', '')
            cookie_categories = cookie_categories.split(',')
            for ccateg in cookie_categories:
                if ccateg in categories:
                    selected_categories.append(ccateg)
        if not len(selected_categories):
            selected_categories = ['general']

        for categ in selected_categories:
            selected_engines.extend({'category': categ,
                                     'name': x.name}
                                    for x in categories[categ])

    results, suggestions = search(query,
                                  request,
                                  selected_engines,
                                  pageno,
                                  lang)

    for result in results:
        if not paging and engines[result['engine']].paging:
            paging = True
        if request_data.get('format', 'html') == 'html':
            if 'content' in result:
                result['content'] = highlight_content(result['content'], query)
            result['title'] = highlight_content(result['title'], query)
        else:
            if 'content' in result:
                result['content'] = html_to_text(result['content']).strip()
            # removing html content and whitespace duplications
            result['title'] = ' '.join(html_to_text(result['title'])
                                       .strip().split())
        if len(result['url']) > 74:
            url_parts = result['url'][:35], result['url'][-35:]
            result['pretty_url'] = '{0}[...]{1}'.format(*url_parts)
        else:
            result['pretty_url'] = result['url']

        for engine in result['engines']:
            if engine in favicons:
                result['favicon'] = engine

    if request_data.get('format') == 'json':
        return Response(json.dumps({'query': query, 'results': results}),
                        mimetype='application/json')
    elif request_data.get('format') == 'csv':
        csv = UnicodeWriter(cStringIO.StringIO())
        keys = ('title', 'url', 'content', 'host', 'engine', 'score')
        if len(results):
            csv.writerow(keys)
            for row in results:
                row['host'] = row['parsed_url'].netloc
                csv.writerow([row.get(key, '') for key in keys])
        csv.stream.seek(0)
        response = Response(csv.stream.read(), mimetype='application/csv')
        content_disp = 'attachment;Filename=searx_-_{0}.csv'.format(query)
        response.headers.add('Content-Disposition', content_disp)
        return response
    elif request_data.get('format') == 'rss':
        response_rss = render(
            'opensearch_response_rss.xml',
            results=results,
            q=request_data['q'],
            number_of_results=len(results),
            base_url=get_base_url()
        )
        return Response(response_rss, mimetype='text/xml')

    return render(
        'results.html',
        results=results,
        q=request_data['q'],
        selected_categories=selected_categories,
        paging=paging,
        pageno=pageno,
        suggestions=suggestions
    )


@app.route('/about', methods=['GET'])
def about():
    """Render about page"""
    return render('about.html')


@app.route('/engines', methods=['GET'])
def list_engines():
    """Render engines page.

    List of all supported engines.
    """
    return render('engines.html',
                  categs=categories.items(),
                  shortcuts={y: x for x, y in engine_shortcuts.items()})


@app.route('/preferences', methods=['GET', 'POST'])
def preferences():
    """Render preferences page.

    Settings that are going to be saved as cookies."""
    lang = None

    if request.cookies.get('language')\
       and request.cookies['language'] in (x[0] for x in language_codes):
        lang = request.cookies['language']

    if request.method == 'POST':
        selected_categories = []
        locale = None
        for pd_name, pd in request.form.items():
            if pd_name.startswith('category_'):
                category = pd_name[9:]
                if not category in categories:
                    continue
                selected_categories.append(category)
            elif pd_name == 'locale' and pd in settings['locales']:
                locale = pd
            elif pd_name == 'language' and (pd == 'all' or
                                            pd in (x[0] for
                                                   x in language_codes)):
                lang = pd

        resp = make_response(redirect('/'))

        if locale:
            # cookie max age: 4 weeks
            resp.set_cookie(
                'locale', locale,
                max_age=60 * 60 * 24 * 7 * 4
            )

        if lang:
            # cookie max age: 4 weeks
            resp.set_cookie(
                'language', lang,
                max_age=60 * 60 * 24 * 7 * 4
            )

        if selected_categories:
            # cookie max age: 4 weeks
            resp.set_cookie(
                'categories', ','.join(selected_categories),
                max_age=60 * 60 * 24 * 7 * 4
            )
        return resp
    return render('preferences.html',
                  locales=settings['locales'],
                  current_locale=get_locale(),
                  current_language=lang or 'all',
                  language_codes=language_codes)


@app.route('/stats', methods=['GET'])
def stats():
    """Render engine statistics page."""
    global categories
    stats = get_engines_stats()
    return render('stats.html', stats=stats)


@app.route('/robots.txt', methods=['GET'])
def robots():
    return Response("""User-agent: *
Allow: /
Allow: /about
Disallow: /stats
Disallow: /engines
""", mimetype='text/plain')


@app.route('/opensearch.xml', methods=['GET'])
def opensearch():
    method = 'post'
    # chrome/chromium only supports HTTP GET....
    if request.headers.get('User-Agent', '').lower().find('webkit') >= 0:
        method = 'get'
    ret = render('opensearch.xml', method=method, host=get_base_url())
    resp = Response(response=ret,
                    status=200,
                    mimetype="application/xml")
    return resp


@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static/img'),
                               'favicon.png',
                               mimetype='image/vnd.microsoft.icon')


def run():
    from gevent import monkey
    monkey.patch_all()

    app.run(
        debug=settings['server']['debug'],
        use_debugger=settings['server']['debug'],
        port=settings['server']['port']
    )


if __name__ == "__main__":
    run()