Browse Source

[enh] add supported_languages on engines and auto-generate languages.py

marc 8 years ago
parent
commit
149802c569

+ 3
- 3
searx/autocomplete.py View File

@@ -81,17 +81,17 @@ def searx_bang(full_query):
81 81
             engine_query = full_query.getSearchQuery()[1:]
82 82
 
83 83
             for lc in language_codes:
84
-                lang_id, lang_name, country = map(str.lower, lc)
84
+                lang_id, lang_name, country, english_name = map(str.lower, lc)
85 85
 
86 86
                 # check if query starts with language-id
87 87
                 if lang_id.startswith(engine_query):
88 88
                     if len(engine_query) <= 2:
89
-                        results.append(':{lang_id}'.format(lang_id=lang_id.split('_')[0]))
89
+                        results.append(':{lang_id}'.format(lang_id=lang_id.split('-')[0]))
90 90
                     else:
91 91
                         results.append(':{lang_id}'.format(lang_id=lang_id))
92 92
 
93 93
                 # check if query starts with language name
94
-                if lang_name.startswith(engine_query):
94
+                if lang_name.startswith(engine_query) or english_name.startswith(engine_query):
95 95
                     results.append(':{lang_name}'.format(lang_name=lang_name))
96 96
 
97 97
                 # check if query starts with country

+ 1
- 0
searx/engines/__init__.py View File

@@ -38,6 +38,7 @@ engine_shortcuts = {}
38 38
 engine_default_args = {'paging': False,
39 39
                        'categories': ['general'],
40 40
                        'language_support': True,
41
+                       'supported_languages': [],
41 42
                        'safesearch': False,
42 43
                        'timeout': settings['outgoing']['request_timeout'],
43 44
                        'shortcut': '-',

+ 3
- 2
searx/engines/archlinux.py View File

@@ -29,8 +29,8 @@ xpath_link = './/div[@class="mw-search-result-heading"]/a'
29 29
 
30 30
 # cut 'en' from 'en_US', 'de' from 'de_CH', and so on
31 31
 def locale_to_lang_code(locale):
32
-    if locale.find('_') >= 0:
33
-        locale = locale.split('_')[0]
32
+    if locale.find('-') >= 0:
33
+        locale = locale.split('-')[0]
34 34
     return locale
35 35
 
36 36
 
@@ -95,6 +95,7 @@ main_langs = {
95 95
     'uk': 'Українська',
96 96
     'zh': '简体中文'
97 97
 }
98
+supported_languages = dict(lang_urls, **main_langs)
98 99
 
99 100
 
100 101
 # do search-request

+ 1
- 1
searx/engines/bing.py View File

@@ -32,7 +32,7 @@ def request(query, params):
32 32
     offset = (params['pageno'] - 1) * 10 + 1
33 33
 
34 34
     if params['language'] != 'all':
35
-        query = u'language:{} {}'.format(params['language'].split('_')[0].upper(),
35
+        query = u'language:{} {}'.format(params['language'].split('-')[0].upper(),
36 36
                                          query.decode('utf-8')).encode('utf-8')
37 37
 
38 38
     search_path = search_string.format(

+ 2
- 1
searx/engines/bing_images.py View File

@@ -19,6 +19,7 @@ from urllib import urlencode
19 19
 from lxml import html
20 20
 from json import loads
21 21
 import re
22
+from searx.engines.bing import supported_languages
22 23
 
23 24
 # engine dependent config
24 25
 categories = ['images']
@@ -53,7 +54,7 @@ def request(query, params):
53 54
     if params['language'] == 'all':
54 55
         language = 'en-US'
55 56
     else:
56
-        language = params['language'].replace('_', '-')
57
+        language = params['language']
57 58
 
58 59
     search_path = search_string.format(
59 60
         query=urlencode({'q': query}),

+ 2
- 1
searx/engines/bing_news.py View File

@@ -17,6 +17,7 @@ from datetime import datetime
17 17
 from dateutil import parser
18 18
 from lxml import etree
19 19
 from searx.utils import list_get
20
+from searx.engines.bing import supported_languages
20 21
 
21 22
 # engine dependent config
22 23
 categories = ['news']
@@ -74,7 +75,7 @@ def request(query, params):
74 75
     if params['language'] == 'all':
75 76
         language = 'en-US'
76 77
     else:
77
-        language = params['language'].replace('_', '-')
78
+        language = params['language']
78 79
 
79 80
     params['url'] = _get_url(query, language, offset, params['time_range'])
80 81
 

+ 40
- 2
searx/engines/duckduckgo.py View File

@@ -22,6 +22,13 @@ from searx.languages import language_codes
22 22
 categories = ['general']
23 23
 paging = True
24 24
 language_support = True
25
+supported_languages = ["es-AR", "en-AU", "de-AT", "fr-BE", "nl-BE", "pt-BR", "bg-BG", "en-CA", "fr-CA", "ca-CT",
26
+                       "es-CL", "zh-CN", "es-CO", "hr-HR", "cs-CZ", "da-DK", "et-EE", "fi-FI", "fr-FR", "de-DE",
27
+                       "el-GR", "tzh-HK", "hu-HU", "en-IN", "id-ID", "en-ID", "en-IE", "he-IL", "it-IT", "jp-JP",
28
+                       "kr-KR", "es-XL", "lv-LV", "lt-LT", "ms-MY", "en-MY", "es-MX", "nl-NL", "en-NZ", "no-NO",
29
+                       "es-PE", "en-PH", "tl-PH", "pl-PL", "pt-PT", "ro-RO", "ru-RU", "ar-XA", "en-XA", "en-SG",
30
+                       "sk-SK", "sl-SL", "en-ZA", "es-ES", "ca-ES", "sv-SE", "de-CH", "fr-CH", "it-CH", "tzh-TW",
31
+                       "th-TH", "tr-TR", "uk-UA", "en-UK", "en-US", "es-US", "vi-VN"]
25 32
 time_range_support = True
26 33
 
27 34
 # search-url
@@ -46,10 +53,23 @@ def request(query, params):
46 53
 
47 54
     offset = (params['pageno'] - 1) * 30
48 55
 
56
+    # custom fixes for languages
49 57
     if params['language'] == 'all':
50 58
         locale = None
59
+    elif params['language'][:2] == 'ja':
60
+        locale = 'jp-jp'
61
+    elif params['language'] == 'zh-TW':
62
+        locale = 'tw-tzh'
63
+    elif params['language'] == 'zh-HK':
64
+        locale = 'hk-tzh'
65
+    elif params['language'][-2:] == 'SA':
66
+        locale = 'xa' + params['language'].split('-')[0]
67
+    elif params['language'][-2:] == 'GB':
68
+        locale = 'uk' + params['language'].split('-')[0]
69
+    elif params['language'] == 'es-419':
70
+        locale = 'xl-es'
51 71
     else:
52
-        locale = params['language'].split('_')
72
+        locale = params['language'].split('-')
53 73
         if len(locale) == 2:
54 74
             # country code goes first
55 75
             locale = locale[1].lower() + '-' + locale[0].lower()
@@ -58,7 +78,25 @@ def request(query, params):
58 78
             locale = locale[0].lower()
59 79
             lang_codes = [x[0] for x in language_codes]
60 80
             for lc in lang_codes:
61
-                lc = lc.split('_')
81
+                lc = lc.split('-')
82
+                if locale == lc[0] and len(lc) == 2:
83
+                    locale = lc[1].lower() + '-' + lc[0].lower()
84
+                    break
85
+
86
+    if locale:
87
+        params['url'] = url.format(
88
+            query=urlencode({'q': query, 'kl': locale}), offset=offset)
89
+    else:
90
+        locale = params['language'].split('-')
91
+        if len(locale) == 2:
92
+            # country code goes first
93
+            locale = locale[1].lower() + '-' + locale[0].lower()
94
+        else:
95
+            # tries to get a country code from language
96
+            locale = locale[0].lower()
97
+            lang_codes = [x[0] for x in language_codes]
98
+            for lc in lang_codes:
99
+                lc = lc.split('-')
62 100
                 if locale == lc[0]:
63 101
                     locale = lc[1].lower() + '-' + lc[0].lower()
64 102
                     break

+ 2
- 1
searx/engines/duckduckgo_definitions.py View File

@@ -4,6 +4,7 @@ from re import compile, sub
4 4
 from lxml import html
5 5
 from searx.utils import html_to_text
6 6
 from searx.engines.xpath import extract_text
7
+from searx.engines.duckduckgo import supported_languages
7 8
 
8 9
 url = 'https://api.duckduckgo.com/'\
9 10
     + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
@@ -23,7 +24,7 @@ def result_to_text(url, text, htmlResult):
23 24
 
24 25
 def request(query, params):
25 26
     params['url'] = url.format(query=urlencode({'q': query}))
26
-    params['headers']['Accept-Language'] = params['language']
27
+    params['headers']['Accept-Language'] = params['language'].split('-')[0]
27 28
     return params
28 29
 
29 30
 

+ 1
- 1
searx/engines/gigablast.py View File

@@ -48,7 +48,7 @@ def request(query, params):
48 48
     if params['language'] == 'all':
49 49
         language = 'xx'
50 50
     else:
51
-        language = params['language'][0:2]
51
+        language = params['language'].split('-')[0]
52 52
 
53 53
     if params['safesearch'] >= 1:
54 54
         safesearch = 1

+ 14
- 0
searx/engines/google.py View File

@@ -23,6 +23,20 @@ categories = ['general']
23 23
 paging = True
24 24
 language_support = True
25 25
 use_locale_domain = True
26
+supported_languages = ['de', 'en', 'es', 'es_419', 'fr', 'hr', 'it', 'nl', 'pl', 'pt-BR',
27
+                       'pt-PT', 'vi', 'tr', 'ru', 'ar', 'th', 'ko', 'zh-CN', 'zh-TW', 'ja',
28
+                       'ach', 'af', 'ak', 'az', 'ms', 'ban', 'xx_bork', 'bs', 'br', 'ca',
29
+                       'ceb', 'ckb', 'cs', 'sn', 'co', 'cy', 'da', 'yo', 'et', 'xx_elmer',
30
+                       'eo', 'eu', 'ee', 'tl', 'fo', 'gaa', 'ga', 'gd', 'gl', 'gn', 'xx_hacker',
31
+                       'ht', 'ha', 'haw', 'bem', 'ig', 'rn', 'id', 'ia', 'zu', 'is', 'jw', 'rw',
32
+                       'sw', 'tlh', 'kg', 'mfe', 'kri', 'la', 'lv', 'to', 'lt', 'ln', 'loz',
33
+                       'lua', 'lg', 'hu', 'mg', 'mt', 'mi', 'pcm', 'no', 'nso', 'ny', 'nn',
34
+                       'uz', 'oc', 'om', 'xx_pirate', 'pt', 'ro', 'mo', 'rm', 'qu', 'nyn', 'crs',
35
+                       'sq', 'sd', 'sk', 'sl', 'so', 'st', 'sr_ME', 'sr_Latn', 'su', 'fi', 'sv',
36
+                       'tg', 'tt', 'tn', 'tum', 'tk', 'tw', 'fy', 'wo', 'xh', 'el', 'be', 'bg',
37
+                       'ky', 'kk', 'mk', 'mn', 'sr', 'uk', 'ka', 'hy', 'yi', 'iw', 'ug', 'ur',
38
+                       'ps', 'fa', 'ti', 'am', 'ne', 'mr', 'hi', 'bn', 'pa', 'gu', 'or', 'ta',
39
+                       'te', 'kn', 'ml', 'si', 'lo', 'my', 'km', 'chr']
26 40
 time_range_support = True
27 41
 
28 42
 # based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests

+ 3
- 1
searx/engines/google_news.py View File

@@ -12,6 +12,8 @@
12 12
 
13 13
 from lxml import html
14 14
 from urllib import urlencode
15
+from json import loads
16
+from searx.engines.google import supported_languages
15 17
 
16 18
 # search-url
17 19
 categories = ['news']
@@ -50,7 +52,7 @@ def request(query, params):
50 52
                                       search_options=urlencode(search_options))
51 53
 
52 54
     if params['language'] != 'all':
53
-        language_array = params['language'].lower().split('_')
55
+        language_array = params['language'].lower().split('-')
54 56
         params['url'] += '&lr=lang_' + language_array[0]
55 57
 
56 58
     return params

+ 2
- 1
searx/engines/mediawiki.py View File

@@ -15,6 +15,7 @@
15 15
 from json import loads
16 16
 from string import Formatter
17 17
 from urllib import urlencode, quote
18
+from searx.engines.wikipedia import supported_engines
18 19
 
19 20
 # engine dependent config
20 21
 categories = ['general']
@@ -46,7 +47,7 @@ def request(query, params):
46 47
     if params['language'] == 'all':
47 48
         language = 'en'
48 49
     else:
49
-        language = params['language'].split('_')[0]
50
+        language = params['language'].split('-')[0]
50 51
 
51 52
     # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]
52 53
     if any(x[1] == 'language' for x in format_strings):

+ 2
- 2
searx/engines/photon.py View File

@@ -26,7 +26,7 @@ search_string = 'api/?{query}&limit={limit}'
26 26
 result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
27 27
 
28 28
 # list of supported languages
29
-allowed_languages = ['de', 'en', 'fr', 'it']
29
+supported_languages = ['de', 'en', 'fr', 'it']
30 30
 
31 31
 
32 32
 # do search-request
@@ -37,7 +37,7 @@ def request(query, params):
37 37
 
38 38
     if params['language'] != 'all':
39 39
         language = params['language'].split('_')[0]
40
-        if language in allowed_languages:
40
+        if language in supported_languages:
41 41
             params['url'] = params['url'] + "&lang=" + language
42 42
 
43 43
     # using searx User-Agent

+ 1
- 1
searx/engines/startpage.py View File

@@ -47,7 +47,7 @@ def request(query, params):
47 47
 
48 48
     # set language if specified
49 49
     if params['language'] != 'all':
50
-        params['data']['with_language'] = ('lang_' + params['language'].split('_')[0])
50
+        params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])
51 51
 
52 52
     return params
53 53
 

+ 7
- 2
searx/engines/subtitleseeker.py View File

@@ -43,8 +43,13 @@ def response(resp):
43 43
 
44 44
     search_lang = ""
45 45
 
46
-    if resp.search_params['language'] != 'all':
47
-        search_lang = [lc[1]
46
+    # dirty fix for languages named differenly in their site
47
+    if resp.search_params['language'][:2] == 'fa':
48
+        search_lang = 'Farsi'
49
+    elif resp.search_params['language'] == 'pt_BR':
50
+        search_lang = 'Brazilian'
51
+    elif resp.search_params['language'] != 'all':
52
+        search_lang = [lc[3]
48 53
                        for lc in language_codes
49 54
                        if lc[0][:2] == resp.search_params['language'].split('_')[0]][0]
50 55
 

+ 2
- 2
searx/engines/swisscows.py View File

@@ -36,8 +36,8 @@ def request(query, params):
36 36
         ui_language = 'browser'
37 37
         region = 'browser'
38 38
     else:
39
-        region = params['language'].replace('_', '-')
40
-        ui_language = params['language'].split('_')[0]
39
+        region = params['language']
40
+        ui_language = params['language'].split('-')[0]
41 41
 
42 42
     search_path = search_string.format(
43 43
         query=urlencode({'query': query,

+ 1
- 1
searx/engines/twitter.py View File

@@ -40,7 +40,7 @@ def request(query, params):
40 40
 
41 41
     # set language if specified
42 42
     if params['language'] != 'all':
43
-        params['cookies']['lang'] = params['language'].split('_')[0]
43
+        params['cookies']['lang'] = params['language'].split('-')[0]
44 44
     else:
45 45
         params['cookies']['lang'] = 'en'
46 46
 

+ 2
- 0
searx/engines/wikidata.py View File

@@ -14,6 +14,8 @@
14 14
 from searx import logger
15 15
 from searx.poolrequests import get
16 16
 from searx.engines.xpath import extract_text
17
+from searx.utils import format_date_by_locale
18
+from searx.engines.wikipedia import supported_languages
17 19
 
18 20
 from json import loads
19 21
 from lxml.html import fromstring

+ 33
- 2
searx/engines/wikipedia.py View File

@@ -13,6 +13,36 @@
13 13
 from json import loads
14 14
 from urllib import urlencode, quote
15 15
 
16
+supported_languages = ["en", "sv", "ceb", "de", "nl", "fr", "ru", "it", "es", "war",
17
+                       "pl", "vi", "ja", "pt", "zh", "uk", "ca", "fa", "no", "sh",
18
+                       "ar", "fi", "hu", "id", "ro", "cs", "ko", "sr", "ms", "tr",
19
+                       "eu", "eo", "min", "bg", "da", "kk", "sk", "hy", "he", "zh-min-nan",
20
+                       "lt", "hr", "sl", "et", "ce", "gl", "nn", "uz", "la", "vo",
21
+                       "el", "simple", "be", "az", "th", "ur", "ka", "hi", "oc", "ta",
22
+                       "mk", "mg", "new", "lv", "cy", "bs", "tt", "tl", "te", "pms",
23
+                       "be-tarask", "br", "sq", "ky", "ht", "jv", "tg", "ast", "zh-yue", "lb",
24
+                       "mr", "ml", "bn", "pnb", "is", "af", "sco", "ga", "ba", "fy",
25
+                       "cv", "lmo", "sw", "my", "an", "yo", "ne", "io", "gu", "nds",
26
+                       "scn", "bpy", "pa", "ku", "als", "kn", "bar", "ia", "qu", "su",
27
+                       "ckb", "bat-smg", "mn", "arz", "nap", "wa", "bug", "gd", "yi", "map-bms",
28
+                       "am", "mzn", "fo", "si", "nah", "li", "sah", "vec", "hsb", "or",
29
+                       "os", "mrj", "sa", "hif", "mhr", "roa-tara", "azb", "pam", "ilo",
30
+                       "sd", "ps", "se", "mi", "bh", "eml", "bcl", "xmf", "diq", "hak",
31
+                       "gan", "glk", "vls", "nds-nl", "rue", "bo", "fiu-vro", "co", "sc",
32
+                       "tk", "csb", "lrc", "vep", "wuu", "km", "szl", "gv", "crh", "kv",
33
+                       "zh-classical", "frr", "zea", "as", "so", "kw", "nso", "ay", "stq",
34
+                       "udm", "cdo", "nrm", "ie", "koi", "rm", "pcd", "myv", "mt", "fur",
35
+                       "ace", "lad", "gn", "lij", "dsb", "dv", "cbk-zam", "ext", "gom",
36
+                       "kab", "ksh", "ang", "mai", "mwl", "lez", "gag", "ln", "ug", "pi",
37
+                       "pag", "frp", "sn", "nv", "av", "pfl", "haw", "xal", "krc", "kaa",
38
+                       "rw", "bxr", "pdc", "to", "kl", "nov", "arc", "kbd", "lo", "bjn",
39
+                       "pap", "ha", "tet", "ki", "tyv", "tpi", "na", "lbe", "ig", "jbo",
40
+                       "roa-rup", "ty", "jam", "za", "kg", "mdf", "lg", "wo", "srn", "ab",
41
+                       "ltg", "zu", "sm", "chr", "om", "tn", "chy", "rmy", "cu", "tw", "tum",
42
+                       "xh", "bi", "rn", "pih", "got", "ss", "pnt", "bm", "ch", "mo", "ts",
43
+                       "ady", "iu", "st", "ee", "ny", "fj", "ks", "ak", "ik", "sg", "ve",
44
+                       "dz", "ff", "ti", "cr", "ng", "cho", "kj", "mh", "ho", "ii", "aa", "mus", "hz", "kr"]
45
+
16 46
 # search-url
17 47
 base_url = 'https://{language}.wikipedia.org/'
18 48
 search_postfix = 'w/api.php?'\
@@ -28,10 +58,11 @@ search_postfix = 'w/api.php?'\
28 58
 
29 59
 # set language in base_url
30 60
 def url_lang(lang):
31
-    if lang == 'all':
61
+    lang = lang.split('-')[0]
62
+    if lang == 'all' or lang not in supported_languages:
32 63
         language = 'en'
33 64
     else:
34
-        language = lang.split('_')[0]
65
+        language = lang
35 66
 
36 67
     return base_url.format(language=language)
37 68
 

+ 1
- 1
searx/engines/yacy.py View File

@@ -53,7 +53,7 @@ def request(query, params):
53 53
 
54 54
     # add language tag if specified
55 55
     if params['language'] != 'all':
56
-        params['url'] += '&lr=lang_' + params['language'].split('_')[0]
56
+        params['url'] += '&lr=lang_' + params['language'].split('-')[0]
57 57
 
58 58
     return params
59 59
 

+ 11
- 1
searx/engines/yahoo.py View File

@@ -20,6 +20,10 @@ from searx.engines.xpath import extract_text, extract_url
20 20
 categories = ['general']
21 21
 paging = True
22 22
 language_support = True
23
+supported_languages = ["ar", "bg", "ca", "szh", "tzh", "hr", "cs", "da", "nl", "en",
24
+                       "et", "fi", "fr", "de", "el", "he", "hu", "is", "id", "it", "ja",
25
+                       "ko", "lv", "lt", "no", "fa", "pl", "pt", "ro", "ru", "sk", "sr",
26
+                       "sl", "es", "sv", "th", "tr"]
23 27
 time_range_support = True
24 28
 
25 29
 # search-url
@@ -72,7 +76,13 @@ def _get_url(query, offset, language, time_range):
72 76
 def _get_language(params):
73 77
     if params['language'] == 'all':
74 78
         return 'en'
75
-    return params['language'].split('_')[0]
79
+    elif params['language'][:2] == 'zh':
80
+        if params['language'] == 'zh' or params['language'] == 'zh-CH':
81
+            return 'szh'
82
+        else:
83
+            return 'tzh'
84
+    else:
85
+        return params['language'].split('-')[0]
76 86
 
77 87
 
78 88
 # do search-request

+ 1
- 1
searx/engines/yahoo_news.py View File

@@ -12,7 +12,7 @@
12 12
 from urllib import urlencode
13 13
 from lxml import html
14 14
 from searx.engines.xpath import extract_text, extract_url
15
-from searx.engines.yahoo import parse_url
15
+from searx.engines.yahoo import parse_url, supported_languages
16 16
 from datetime import datetime, timedelta
17 17
 import re
18 18
 from dateutil import parser

+ 1
- 1
searx/engines/yandex.py View File

@@ -36,7 +36,7 @@ content_xpath = './/div[@class="text-container typo typo_text_m typo_line_m orga
36 36
 
37 37
 
38 38
 def request(query, params):
39
-    lang = params['language'].split('_')[0]
39
+    lang = params['language'].split('-')[0]
40 40
     host = base_url.format(tld=language_map.get(lang) or default_tld)
41 41
     params['url'] = host + search_url.format(page=params['pageno'] - 1,
42 42
                                              query=urlencode({'text': query}))

+ 1
- 1
searx/engines/youtube_api.py View File

@@ -36,7 +36,7 @@ def request(query, params):
36 36
 
37 37
     # add language tag if specified
38 38
     if params['language'] != 'all':
39
-        params['url'] += '&relevanceLanguage=' + params['language'].split('_')[0]
39
+        params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]
40 40
 
41 41
     return params
42 42
 

+ 388
- 76
searx/languages.py View File

@@ -1,78 +1,390 @@
1
-'''
2
-searx is free software: you can redistribute it and/or modify
3
-it under the terms of the GNU Affero General Public License as published by
4
-the Free Software Foundation, either version 3 of the License, or
5
-(at your option) any later version.
6
-
7
-searx is distributed in the hope that it will be useful,
8
-but WITHOUT ANY WARRANTY; without even the implied warranty of
9
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10
-GNU Affero General Public License for more details.
11
-
12
-You should have received a copy of the GNU Affero General Public License
13
-along with searx. If not, see < http://www.gnu.org/licenses/ >.
14
-
15
-(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
16
-'''
17
-
1
+# -*- coding: utf-8 -*-
18 2
 # list of language codes
3
+# this file is generated automatically by utils/update_search_languages.py
4
+
19 5
 language_codes = (
20
-    ("ar_XA", "Arabic", "Arabia"),
21
-    ("bg_BG", "Bulgarian", "Bulgaria"),
22
-    ("cs_CZ", "Czech", "Czech Republic"),
23
-    ("da_DK", "Danish", "Denmark"),
24
-    ("de_AT", "German", "Austria"),
25
-    ("de_CH", "German", "Switzerland"),
26
-    ("de_DE", "German", "Germany"),
27
-    ("el_GR", "Greek", "Greece"),
28
-    ("en_AU", "English", "Australia"),
29
-    ("en_CA", "English", "Canada"),
30
-    ("en_GB", "English", "United Kingdom"),
31
-    ("en_ID", "English", "Indonesia"),
32
-    ("en_IE", "English", "Ireland"),
33
-    ("en_IN", "English", "India"),
34
-    ("en_MY", "English", "Malaysia"),
35
-    ("en_NZ", "English", "New Zealand"),
36
-    ("en_PH", "English", "Philippines"),
37
-    ("en_SG", "English", "Singapore"),
38
-    ("en_US", "English", "United States"),
39
-    ("en_XA", "English", "Arabia"),
40
-    ("en_ZA", "English", "South Africa"),
41
-    ("es_AR", "Spanish", "Argentina"),
42
-    ("es_CL", "Spanish", "Chile"),
43
-    ("es_ES", "Spanish", "Spain"),
44
-    ("es_MX", "Spanish", "Mexico"),
45
-    ("es_US", "Spanish", "United States"),
46
-    ("es_XL", "Spanish", "Latin America"),
47
-    ("et_EE", "Estonian", "Estonia"),
48
-    ("fi_FI", "Finnish", "Finland"),
49
-    ("fr_BE", "French", "Belgium"),
50
-    ("fr_CA", "French", "Canada"),
51
-    ("fr_CH", "French", "Switzerland"),
52
-    ("fr_FR", "French", "France"),
53
-    ("he_IL", "Hebrew", "Israel"),
54
-    ("hr_HR", "Croatian", "Croatia"),
55
-    ("hu_HU", "Hungarian", "Hungary"),
56
-    ("it_IT", "Italian", "Italy"),
57
-    ("ja_JP", "Japanese", "Japan"),
58
-    ("ko_KR", "Korean", "Korea"),
59
-    ("lt_LT", "Lithuanian", "Lithuania"),
60
-    ("lv_LV", "Latvian", "Latvia"),
61
-    ("nb_NO", "Norwegian", "Norway"),
62
-    ("nl_BE", "Dutch", "Belgium"),
63
-    ("nl_NL", "Dutch", "Netherlands"),
64
-    ("oc_OC", "Occitan", "Occitan"),
65
-    ("pl_PL", "Polish", "Poland"),
66
-    ("pt_BR", "Portuguese", "Brazil"),
67
-    ("pt_PT", "Portuguese", "Portugal"),
68
-    ("ro_RO", "Romanian", "Romania"),
69
-    ("ru_RU", "Russian", "Russia"),
70
-    ("sk_SK", "Slovak", "Slovak Republic"),
71
-    ("sl_SL", "Slovenian", "Slovenia"),
72
-    ("sv_SE", "Swedish", "Sweden"),
73
-    ("th_TH", "Thai", "Thailand"),
74
-    ("tr_TR", "Turkish", "Turkey"),
75
-    ("uk_UA", "Ukrainian", "Ukraine"),
76
-    ("zh_CN", "Chinese", "China"),
77
-    ("zh_HK", "Chinese", "Hong Kong SAR"),
78
-    ("zh_TW", "Chinese", "Taiwan"))
6
+    (u"gv", u"Gaelg", u"", u"Manx"),
7
+    (u"sco", u"Scots", u"", u"Scots"),
8
+    (u"scn", u"Sicilianu", u"", u"Sicilian"),
9
+    (u"gu", u"ગુજરાતી", u"", u"Gujarati"),
10
+    (u"gd", u"Gàidhlig", u"", u"Scottish Gaelic"),
11
+    (u"ga", u"Gaeilge", u"", u"Irish"),
12
+    (u"gn", u"Avañe'ẽ", u"", u"Guarani"),
13
+    (u"gl", u"Galego", u"", u"Galician"),
14
+    (u"als", u"Alemannisch", u"", u"Alemannic"),
15
+    (u"lt-LT", u"Lietuvių", u"", u"Lithuanian"),
16
+    (u"vep", u"Vepsän", u"", u"Vepsian"),
17
+    (u"ty", u"Reo Mā`ohi", u"", u"Tahitian"),
18
+    (u"tw", u"Twi", u"", u"Twi"),
19
+    (u"tt", u"Tatarça / Татарча", u"", u"Tatar"),
20
+    (u"tr", u"Türkçe", u"", u"Turkish"),
21
+    (u"ts", u"Xitsonga", u"", u"Tsonga"),
22
+    (u"tn", u"Setswana", u"", u"Tswana"),
23
+    (u"to", u"faka Tonga", u"", u"Tongan"),
24
+    (u"tl", u"Tagalog", u"", u"Tagalog"),
25
+    (u"vec", u"Vèneto", u"", u"Venetian"),
26
+    (u"th", u"ไทย", u"", u"Thai"),
27
+    (u"ti", u"ትግርኛ", u"", u"Tigrinya"),
28
+    (u"tg", u"Тоҷикӣ", u"", u"Tajik"),
29
+    (u"te", u"తెలుగు", u"", u"Telugu"),
30
+    (u"ta", u"தமிழ்", u"", u"Tamil"),
31
+    (u"lrc", u"لۊری شومالی", u"", u"Northern Luri"),
32
+    (u"en-NZ", u"English", u"", u"English"),
33
+    (u"got", u"𐌲𐌿𐍄𐌹𐍃𐌺", u"", u"Gothic"),
34
+    (u"vls", u"West-Vlams", u"", u"West Flemish"),
35
+    (u"ro", u"Română", u"", u"Romanian"),
36
+    (u"bxr", u"Буряад", u"", u"Buryat"),
37
+    (u"fiu-vro", u"Võro", u"", u"Võro"),
38
+    (u"diq", u"Zazaki", u"", u"Zazaki"),
39
+    (u"zh", u"中文", u"", u"Chinese"),
40
+    (u"pms", u"Piemontèis", u"", u"Piedmontese"),
41
+    (u"za", u"Cuengh", u"", u"Zhuang"),
42
+    (u"zh-HK", u"中文", u"", u"Chinese"),
43
+    (u"zu", u"isiZulu", u"", u"Zulu"),
44
+    (u"tet", u"Tetun", u"", u"Tetum"),
45
+    (u"es-PE", u"Español", u"", u"Spanish"),
46
+    (u"new", u"नेपाल भाषा", u"", u"Newar"),
47
+    (u"lez", u"Лезги чІал (Lezgi č’al)", u"", u"Lezgian"),
48
+    (u"glk", u"گیلکی", u"", u"Gilaki"),
49
+    (u"ko-KR", u"한국어", u"", u"Korean"),
50
+    (u"id-ID", u"Bahasa Indonesia", u"", u"Indonesian"),
51
+    (u"cho", u"Choctaw", u"", u"Choctaw"),
52
+    (u"chr", u"ᏣᎳᎩ", u"", u"Cherokee"),
53
+    (u"vi", u"Tiếng Việt", u"", u"Vietnamese"),
54
+    (u"chy", u"Tsetsêhestâhese", u"", u"Cheyenne"),
55
+    (u"is", u"Íslenska", u"", u"Icelandic"),
56
+    (u"tk", u"تركمن / Туркмен", u"", u"Turkmen"),
57
+    (u"da-DK", u"Dansk", u"", u"Danish"),
58
+    (u"pfl", u"Pälzisch", u"", u"Palatinate German"),
59
+    (u"hu-HU", u"Magyar", u"", u"Hungarian"),
60
+    (u"he-IL", u"עברית", u"", u"Hebrew"),
61
+    (u"mg", u"Malagasy", u"", u"Malagasy"),
62
+    (u"ml", u"മലയാളം", u"", u"Malayalam"),
63
+    (u"mo", u"Молдовеняскэ", u"", u"Moldovan"),
64
+    (u"mn", u"Монгол", u"", u"Mongolian"),
65
+    (u"mi", u"Māori", u"", u"Maori"),
66
+    (u"mh", u"Ebon", u"", u"Marshallese"),
67
+    (u"mk", u"Македонски", u"", u"Macedonian"),
68
+    (u"mt", u"Malti", u"", u"Maltese"),
69
+    (u"ms", u"Bahasa Melayu", u"", u"Malay"),
70
+    (u"mr", u"मराठी", u"", u"Marathi"),
71
+    (u"mwl", u"Mirandés", u"", u"Mirandese"),
72
+    (u"my", u"မြန်မာဘာသာ", u"", u"Burmese"),
73
+    (u"en-PH", u"English", u"", u"English"),
74
+    (u"srn", u"Sranantongo", u"", u"Sranan"),
75
+    (u"pl-PL", u"Polski", u"", u"Polish"),
76
+    (u"sl-SL", u"Slovenščina", u"", u"Slovenian"),
77
+    (u"csb", u"Kaszëbsczi", u"", u"Kashubian"),
78
+    (u"cbk-zam", u"Chavacano de Zamboanga", u"", u"Zamboanga Chavacano"),
79
+    (u"nyn", u"Runyankore", u"", u""),
80
+    (u"ig", u"Igbo", u"", u"Igbo"),
81
+    (u"fr", u"Français", u"", u"French"),
82
+    (u"lad", u"Dzhudezmo", u"", u"Ladino"),
83
+    (u"fy", u"Frysk", u"", u"West Frisian"),
84
+    (u"fa", u"فارسی", u"", u"Persian"),
85
+    (u"ff", u"Fulfulde", u"", u"Fula"),
86
+    (u"mai", u"मैथिली", u"", u"Maithili"),
87
+    (u"fi", u"Suomi", u"", u"Finnish"),
88
+    (u"fj", u"Na Vosa Vakaviti", u"", u"Fijian"),
89
+    (u"fo", u"Føroyskt", u"", u"Faroese"),
90
+    (u"ss", u"SiSwati", u"", u"Swati"),
91
+    (u"roa-tara", u"Tarandíne", u"", u"Tarantino"),
92
+    (u"sq", u"Shqip", u"", u"Albanian"),
93
+    (u"sw", u"Kiswahili", u"", u"Swahili"),
94
+    (u"sv", u"Svenska", u"", u"Swedish"),
95
+    (u"su", u"Basa Sunda", u"", u"Sundanese"),
96
+    (u"st", u"Sesotho", u"", u"Sesotho"),
97
+    (u"sk", u"Slovenčina", u"", u"Slovak"),
98
+    (u"si", u"සිංහල", u"", u"Sinhalese"),
99
+    (u"sh", u"Srpskohrvatski / Српскохрватски", u"", u"Serbo-Croatian"),
100
+    (u"so", u"Soomaali", u"", u"Somali"),
101
+    (u"sn", u"chiShona", u"", u"Shona"),
102
+    (u"sm", u"Gagana Samoa", u"", u"Samoan"),
103
+    (u"sl", u"Slovenščina", u"", u"Slovenian"),
104
+    (u"sc", u"Sardu", u"", u"Sardinian"),
105
+    (u"pt-BR", u"português (Brasil)", u"", u""),
106
+    (u"sa", u"संस्कृतम्", u"", u"Sanskrit"),
107
+    (u"sg", u"Sängö", u"", u"Sango"),
108
+    (u"se", u"Sámegiella", u"", u"Northern Sami"),
109
+    (u"sd", u"سنڌي، سندھی ، सिन्ध", u"", u"Sindhi"),
110
+    (u"fr-CH", u"Français", u"", u"French"),
111
+    (u"zea", u"Zeêuws", u"", u"Zeelandic"),
112
+    (u"it-CH", u"Italiano", u"", u"Italian"),
113
+    (u"wuu", u"吴语", u"", u"Wu"),
114
+    (u"fr-CA", u"Français", u"", u"French"),
115
+    (u"ar-XA", u"العربية", u"", u"Arabic"),
116
+    (u"kbd", u"Адыгэбзэ (Adighabze)", u"", u"Kabardian Circassian"),
117
+    (u"no-NO", u"Norsk (Bokmål)", u"", u"Norwegian (Bokmål)"),
118
+    (u"ca-ES", u"Català", u"", u"Catalan"),
119
+    (u"lg", u"Luganda", u"", u"Luganda"),
120
+    (u"lb", u"Lëtzebuergesch", u"", u"Luxembourgish"),
121
+    (u"la", u"Latina", u"", u"Latin"),
122
+    (u"ln", u"Lingala", u"", u"Lingala"),
123
+    (u"lo", u"ລາວ", u"", u"Lao"),
124
+    (u"de-CH", u"Deutsch", u"", u"German"),
125
+    (u"li", u"Limburgs", u"", u"Limburgish"),
126
+    (u"lv", u"Latviešu", u"", u"Latvian"),
127
+    (u"lt", u"Lietuvių", u"", u"Lithuanian"),
128
+    (u"pcm", u"Nigerian Pidgin", u"", u""),
129
+    (u"pcd", u"Picard", u"", u"Picard"),
130
+    (u"yi", u"ייִדיש", u"", u"Yiddish"),
131
+    (u"ceb", u"Sinugboanong Binisaya", u"", u"Cebuano"),
132
+    (u"yo", u"Yorùbá", u"", u"Yoruba"),
133
+    (u"ro-RO", u"Română", u"", u"Romanian"),
134
+    (u"bar", u"Boarisch", u"", u"Bavarian"),
135
+    (u"nov", u"Novial", u"", u"Novial"),
136
+    (u"sr-ME", u"srpski (Crna Gora)", u"", u""),
137
+    (u"es-CL", u"Español", u"", u"Spanish"),
138
+    (u"es-CO", u"Español", u"", u"Spanish"),
139
+    (u"nl-NL", u"Nederlands", u"", u"Dutch"),
140
+    (u"map-bms", u"Basa Banyumasan", u"", u"Banyumasan"),
141
+    (u"el", u"Ελληνικά", u"", u"Greek"),
142
+    (u"eo", u"Esperanto", u"", u"Esperanto"),
143
+    (u"en", u"English", u"", u"English"),
144
+    (u"ee", u"Eʋegbe", u"", u"Ewe"),
145
+    (u"mdf", u"Мокшень (Mokshanj Kälj)", u"", u"Moksha"),
146
+    (u"eu", u"Euskara", u"", u"Basque"),
147
+    (u"et", u"Eesti", u"", u"Estonian"),
148
+    (u"es", u"Español", u"", u"Spanish"),
149
+    (u"gom", u"गोवा कोंकणी / Gova Konknni", u"", u"Goan Konkani"),
150
+    (u"ru", u"Русский", u"", u"Russian"),
151
+    (u"rw", u"Ikinyarwanda", u"", u"Kinyarwanda"),
152
+    (u"rm", u"Rumantsch", u"", u"Romansh"),
153
+    (u"rn", u"Kirundi", u"", u"Kirundi"),
154
+    (u"es-419", u"español (Latinoamérica)", u"", u""),
155
+    (u"dsb", u"Dolnoserbski", u"", u"Lower Sorbian"),
156
+    (u"ast", u"Asturianu", u"", u"Asturian"),
157
+    (u"lmo", u"Lumbaart", u"", u"Lombard"),
158
+    (u"ltg", u"Latgaļu", u"", u"Latgalian"),
159
+    (u"xh", u"isiXhosa", u"", u"Xhosa"),
160
+    (u"en-CA", u"English", u"", u"English"),
161
+    (u"koi", u"Перем Коми (Perem Komi)", u"", u"Komi-Permyak"),
162
+    (u"tr-TR", u"Türkçe", u"", u"Turkish"),
163
+    (u"pnt", u"Ποντιακά", u"", u"Pontic"),
164
+    (u"es-XL", u"Español", u"", u"Spanish"),
165
+    (u"fi-FI", u"Suomi", u"", u"Finnish"),
166
+    (u"pnb", u"شاہ مکھی پنجابی (Shāhmukhī Pañjābī)", u"", u"Western Punjabi"),
167
+    (u"udm", u"Удмурт кыл", u"", u"Udmurt"),
168
+    (u"bem", u"Ichibemba", u"", u""),
169
+    (u"roa-rup", u"Armãneashce", u"", u"Aromanian"),
170
+    (u"sr-Latn", u"srpski (latinica)", u"", u""),
171
+    (u"stq", u"Seeltersk", u"", u"Saterland Frisian"),
172
+    (u"sr", u"Српски / Srpski", u"", u"Serbian"),
173
+    (u"ang", u"Englisc", u"", u"Anglo-Saxon"),
174
+    (u"ru-RU", u"Русский", u"", u"Russian"),
175
+    (u"lbe", u"Лакку", u"", u"Lak"),
176
+    (u"min", u"Minangkabau", u"", u"Minangkabau"),
177
+    (u"es-US", u"Español", u"", u"Spanish"),
178
+    (u"lij", u"Líguru", u"", u"Ligurian"),
179
+    (u"kab", u"Taqbaylit", u"", u"Kabyle"),
180
+    (u"kaa", u"Qaraqalpaqsha", u"", u"Karakalpak"),
181
+    (u"fr-FR", u"Français", u"", u"French"),
182
+    (u"tyv", u"Тыва", u"", u"Tuvan"),
183
+    (u"ka", u"ქართული", u"", u"Georgian"),
184
+    (u"kg", u"KiKongo", u"", u"Kongo"),
185
+    (u"ckb", u"Soranî / کوردی", u"", u"Sorani"),
186
+    (u"kk", u"Қазақша", u"", u"Kazakh"),
187
+    (u"kj", u"Kuanyama", u"", u"Kuanyama"),
188
+    (u"ki", u"Gĩkũyũ", u"", u"Kikuyu"),
189
+    (u"ko", u"한국어", u"", u"Korean"),
190
+    (u"kn", u"ಕನ್ನಡ", u"", u"Kannada"),
191
+    (u"tpi", u"Tok Pisin", u"", u"Tok Pisin"),
192
+    (u"kl", u"Kalaallisut", u"", u"Greenlandic"),
193
+    (u"ks", u"कश्मीरी / كشميري", u"", u"Kashmiri"),
194
+    (u"kr", u"Kanuri", u"", u"Kanuri"),
195
+    (u"ext", u"Estremeñu", u"", u"Extremaduran"),
196
+    (u"kw", u"Kernewek/Karnuack", u"", u"Cornish"),
197
+    (u"kv", u"Коми", u"", u"Komi"),
198
+    (u"mrj", u"Кырык Мары (Kyryk Mary)", u"", u"Hill Mari"),
199
+    (u"ky", u"Кыргызча", u"", u"Kirghiz"),
200
+    (u"szl", u"Ślůnski", u"", u"Silesian"),
201
+    (u"cdo", u"Mìng-dĕ̤ng-ngṳ̄", u"", u"Min Dong"),
202
+    (u"en-GB", u"English", u"", u"English"),
203
+    (u"xmf", u"მარგალური (Margaluri)", u"", u"Mingrelian"),
204
+    (u"jam", u"Jamaican Creole English", u"", u"Patois"),
205
+    (u"ar-SA", u"العربية", u"", u"Arabic"),
206
+    (u"ksh", u"Ripoarisch", u"", u"Ripuarian"),
207
+    (u"ms-MY", u"Bahasa Melayu", u"", u"Malay"),
208
+    (u"de", u"Deutsch", u"", u"German"),
209
+    (u"da", u"Dansk", u"", u"Danish"),
210
+    (u"dz", u"ཇོང་ཁ", u"", u"Dzongkha"),
211
+    (u"hif", u"Fiji Hindi", u"", u"Fiji Hindi"),
212
+    (u"dv", u"ދިވެހިބަސް", u"", u"Divehi"),
213
+    (u"crs", u"Seychellois Creole", u"", u""),
214
+    (u"qu", u"Runa Simi", u"", u"Quechua"),
215
+    (u"eml", u"Emiliàn e rumagnòl", u"", u"Emilian-Romagnol"),
216
+    (u"ban", u"Balinese", u"", u""),
217
+    (u"crh", u"Qırımtatarca", u"", u"Crimean Tatar"),
218
+    (u"arz", u"مصرى (Maṣri)", u"", u"Egyptian Arabic"),
219
+    (u"rmy", u"romani - रोमानी", u"", u"Romani"),
220
+    (u"arc", u"ܐܪܡܝܐ", u"", u"Aramaic"),
221
+    (u"th-TH", u"ไทย", u"", u"Thai"),
222
+    (u"mus", u"Muskogee", u"", u"Muscogee"),
223
+    (u"lua", u"Luba-Lulua", u"", u""),
224
+    (u"en-ZA", u"English", u"", u"English"),
225
+    (u"wa", u"Walon", u"", u"Walloon"),
226
+    (u"wo", u"Wolof", u"", u"Wolof"),
227
+    (u"jv", u"Basa Jawa", u"", u"Javanese"),
228
+    (u"jw", u"Javanese", u"", u""),
229
+    (u"fr-BE", u"Français", u"", u"French"),
230
+    (u"tum", u"chiTumbuka", u"", u"Tumbuka"),
231
+    (u"ja", u"日本語", u"", u"Japanese"),
232
+    (u"pt-PT", u"português (Portugal)", u"", u""),
233
+    (u"ilo", u"Ilokano", u"", u"Ilokano"),
234
+    (u"tlh", u"Klingon", u"", u""),
235
+    (u"pdc", u"Deitsch", u"", u"Pennsylvania German"),
236
+    (u"aa", u"Afar", u"", u"Afar"),
237
+    (u"ch", u"Chamoru", u"", u"Chamorro"),
238
+    (u"co", u"Corsu", u"", u"Corsican"),
239
+    (u"simple", u"Simple English", u"", u"Simple English"),
240
+    (u"ca", u"Català", u"", u"Catalan"),
241
+    (u"xx-pirate", u"Pirate", u"", u""),
242
+    (u"ce", u"Нохчийн", u"", u"Chechen"),
243
+    (u"cy", u"Cymraeg", u"", u"Welsh"),
244
+    (u"sah", u"Саха тыла (Saxa Tyla)", u"", u"Sakha"),
245
+    (u"cs", u"Čeština", u"", u"Czech"),
246
+    (u"cr", u"Nehiyaw", u"", u"Cree"),
247
+    (u"bg-BG", u"Български", u"", u"Bulgarian"),
248
+    (u"cv", u"Чăваш", u"", u"Chuvash"),
249
+    (u"cu", u"Словѣньскъ", u"", u"Old Church Slavonic"),
250
+    (u"ps", u"پښتو", u"", u"Pashto"),
251
+    (u"pt", u"Português", u"", u"Portuguese"),
252
+    (u"vi-VN", u"Tiếng Việt", u"", u"Vietnamese"),
253
+    (u"frr", u"Nordfriisk", u"", u"North Frisian"),
254
+    (u"frp", u"Arpitan", u"", u"Franco-Provençal"),
255
+    (u"xal", u"Хальмг", u"", u"Kalmyk"),
256
+    (u"pi", u"पाऴि", u"", u"Pali"),
257
+    (u"it-IT", u"Italiano", u"", u"Italian"),
258
+    (u"pl", u"Polski", u"", u"Polish"),
259
+    (u"nrm", u"Nouormand/Normaund", u"", u"Norman"),
260
+    (u"en-US", u"English", u"", u"English"),
261
+    (u"gan", u"贛語", u"", u"Gan"),
262
+    (u"bat-smg", u"Žemaitėška", u"", u"Samogitian"),
263
+    (u"en-UK", u"English", u"", u"English"),
264
+    (u"gag", u"Gagauz", u"", u"Gagauz"),
265
+    (u"an", u"Aragonés", u"", u"Aragonese"),
266
+    (u"gaa", u"Ga", u"", u""),
267
+    (u"fur", u"Furlan", u"", u"Friulian"),
268
+    (u"kr-KR", u"Kanuri", u"", u"Kanuri"),
269
+    (u"zh-CN", u"中文 (简体)", u"", u""),
270
+    (u"tl-PH", u"Tagalog", u"", u"Tagalog"),
271
+    (u"en-IN", u"English", u"", u"English"),
272
+    (u"ve", u"Tshivenda", u"", u"Venda"),
273
+    (u"en-ID", u"English", u"", u"English"),
274
+    (u"en-IE", u"English", u"", u"English"),
275
+    (u"xx-bork", u"Bork, bork, bork!", u"", u""),
276
+    (u"iu", u"ᐃᓄᒃᑎᑐᑦ", u"", u"Inuktitut"),
277
+    (u"it", u"Italiano", u"", u"Italian"),
278
+    (u"iw", u"עברית", u"", u""),
279
+    (u"vo", u"Volapük", u"", u"Volapük"),
280
+    (u"ii", u"ꆇꉙ", u"", u"Sichuan Yi"),
281
+    (u"ik", u"Iñupiak", u"", u"Inupiak"),
282
+    (u"io", u"Ido", u"", u"Ido"),
283
+    (u"ia", u"Interlingua", u"", u"Interlingua"),
284
+    (u"ja-JP", u"日本語", u"", u"Japanese"),
285
+    (u"ie", u"Interlingue", u"", u"Interlingue"),
286
+    (u"id", u"Bahasa Indonesia", u"", u"Indonesian"),
287
+    (u"nds-nl", u"Nedersaksisch", u"", u"Dutch Low Saxon"),
288
+    (u"pap", u"Papiamentu", u"", u"Papiamentu"),
289
+    (u"pag", u"Pangasinan", u"", u"Pangasinan"),
290
+    (u"pam", u"Kapampangan", u"", u"Kapampangan"),
291
+    (u"lv-LV", u"Latviešu", u"", u"Latvian"),
292
+    (u"mzn", u"مَزِروني", u"", u"Mazandarani"),
293
+    (u"nl-BE", u"Nederlands", u"", u"Dutch"),
294
+    (u"sk-SK", u"Slovenčina", u"", u"Slovak"),
295
+    (u"zh-TW", u"中文 (繁體)", u"", u""),
296
+    (u"es-MX", u"Español", u"", u"Spanish"),
297
+    (u"de-DE", u"Deutsch", u"", u"German"),
298
+    (u"jbo", u"Lojban", u"", u"Lojban"),
299
+    (u"mfe", u"kreol morisien", u"", u""),
300
+    (u"hak", u"Hak-kâ-fa / 客家話", u"", u"Hakka"),
301
+    (u"ny", u"Chichewa", u"", u"Chichewa"),
302
+    (u"ady", u"Адыгэбзэ", u"", u"Adyghe"),
303
+    (u"haw", u"Hawai`i", u"", u"Hawaiian"),
304
+    (u"el-GR", u"Ελληνικά", u"", u"Greek"),
305
+    (u"bpy", u"ইমার ঠার/বিষ্ণুপ্রিয়া মণিপুরী", u"", u"Bishnupriya Manipuri"),
306
+    (u"mhr", u"Олык Марий (Olyk Marij)", u"", u"Meadow Mari"),
307
+    (u"ca-CT", u"Català", u"", u"Catalan"),
308
+    (u"en-MY", u"English", u"", u"English"),
309
+    (u"sv-SE", u"Svenska", u"", u"Swedish"),
310
+    (u"de-AT", u"Deutsch", u"", u"German"),
311
+    (u"xx-elmer", u"Elmer Fudd", u"", u""),
312
+    (u"hsb", u"Hornjoserbsce", u"", u"Upper Sorbian"),
313
+    (u"be", u"Беларуская", u"", u"Belarusian"),
314
+    (u"bg", u"Български", u"", u"Bulgarian"),
315
+    (u"ba", u"Башҡорт", u"", u"Bashkir"),
316
+    (u"bm", u"Bamanankan", u"", u"Bambara"),
317
+    (u"bn", u"বাংলা", u"", u"Bengali"),
318
+    (u"bo", u"བོད་སྐད", u"", u"Tibetan"),
319
+    (u"bh", u"भोजपुरी", u"", u"Bihari"),
320
+    (u"bi", u"Bislama", u"", u"Bislama"),
321
+    (u"rue", u"Русиньскый", u"", u"Rusyn"),
322
+    (u"et-EE", u"Eesti", u"", u"Estonian"),
323
+    (u"br", u"Brezhoneg", u"", u"Breton"),
324
+    (u"bs", u"Bosanski", u"", u"Bosnian"),
325
+    (u"om", u"Oromoo", u"", u"Oromo"),
326
+    (u"ace", u"Bahsa Acèh", u"", u"Acehnese"),
327
+    (u"es-AR", u"Español", u"", u"Spanish"),
328
+    (u"ach", u"Acoli", u"", u""),
329
+    (u"oc", u"Occitan", u"", u"Occitan"),
330
+    (u"kri", u"Krio (Sierra Leone)", u"", u""),
331
+    (u"be-tarask", u"Беларуская (тарашкевіца)", u"", u"Belarusian (Taraškievica)"),
332
+    (u"krc", u"Къарачай-Малкъар (Qarachay-Malqar)", u"", u"Karachay-Balkar"),
333
+    (u"nds", u"Plattdüütsch", u"", u"Low Saxon"),
334
+    (u"os", u"Иронау", u"", u"Ossetian"),
335
+    (u"or", u"ଓଡ଼ିଆ", u"", u"Oriya"),
336
+    (u"nso", u"Sepedi", u"", u"Northern Sotho"),
337
+    (u"bjn", u"Bahasa Banjar", u"", u"Banjar"),
338
+    (u"xx-hacker", u"Hacker", u"", u""),
339
+    (u"zh-min-nan", u"Bân-lâm-gú", u"", u"Min Nan"),
340
+    (u"pa", u"ਪੰਜਾਬੀ", u"", u"Punjabi"),
341
+    (u"loz", u"Lozi", u"", u""),
342
+    (u"war", u"Winaray", u"", u"Waray-Waray"),
343
+    (u"hz", u"Otsiherero", u"", u"Herero"),
344
+    (u"hy", u"Հայերեն", u"", u"Armenian"),
345
+    (u"hr", u"Hrvatski", u"", u"Croatian"),
346
+    (u"ht", u"Krèyol ayisyen", u"", u"Haitian"),
347
+    (u"hu", u"Magyar", u"", u"Hungarian"),
348
+    (u"hi", u"हिन्दी", u"", u"Hindi"),
349
+    (u"ho", u"Hiri Motu", u"", u"Hiri Motu"),
350
+    (u"ha", u"هَوُسَ", u"", u"Hausa"),
351
+    (u"bug", u"Basa Ugi", u"", u"Buginese"),
352
+    (u"he", u"עברית", u"", u"Hebrew"),
353
+    (u"hr-HR", u"Hrvatski", u"", u"Croatian"),
354
+    (u"uz", u"O‘zbek", u"", u"Uzbek"),
355
+    (u"azb", u"تۆرکجه", u"", u"South Azerbaijani"),
356
+    (u"ur", u"اردو", u"", u"Urdu"),
357
+    (u"uk", u"Українська", u"", u"Ukrainian"),
358
+    (u"ug", u"ئۇيغۇر تىلى", u"", u"Uyghur"),
359
+    (u"pih", u"Norfuk", u"", u"Norfolk"),
360
+    (u"ab", u"Аҧсуа", u"", u"Abkhazian"),
361
+    (u"af", u"Afrikaans", u"", u"Afrikaans"),
362
+    (u"ak", u"Akana", u"", u"Akan"),
363
+    (u"am", u"አማርኛ", u"", u"Amharic"),
364
+    (u"myv", u"Эрзянь (Erzjanj Kelj)", u"", u"Erzya"),
365
+    (u"as", u"অসমীয়া", u"", u"Assamese"),
366
+    (u"ar", u"العربية", u"", u"Arabic"),
367
+    (u"km", u"ភាសាខ្មែរ", u"", u"Khmer"),
368
+    (u"uk-UA", u"Українська", u"", u"Ukrainian"),
369
+    (u"av", u"Авар", u"", u"Avar"),
370
+    (u"ay", u"Aymar", u"", u"Aymara"),
371
+    (u"az", u"Azərbaycanca", u"", u"Azerbaijani"),
372
+    (u"es-ES", u"Español", u"", u"Spanish"),
373
+    (u"nl", u"Nederlands", u"", u"Dutch"),
374
+    (u"nn", u"Nynorsk", u"", u"Norwegian (Nynorsk)"),
375
+    (u"no", u"Norsk (Bokmål)", u"", u"Norwegian (Bokmål)"),
376
+    (u"na", u"dorerin Naoero", u"", u"Nauruan"),
377
+    (u"nah", u"Nāhuatl", u"", u"Nahuatl"),
378
+    (u"ne", u"नेपाली", u"", u"Nepali"),
379
+    (u"ng", u"Oshiwambo", u"", u"Ndonga"),
380
+    (u"en-AU", u"English", u"", u"English"),
381
+    (u"nap", u"Nnapulitano", u"", u"Neapolitan"),
382
+    (u"nv", u"Diné bizaad", u"", u"Navajo"),
383
+    (u"ku", u"Kurdî / كوردی", u"", u"Kurdish"),
384
+    (u"cs-CZ", u"Čeština", u"", u"Czech"),
385
+    (u"zh-yue", u"粵語", u"", u"Cantonese"),
386
+    (u"en-SG", u"English", u"", u"English"),
387
+    (u"zh-classical", u"古文 / 文言文", u"", u"Classical Chinese"),
388
+    (u"bcl", u"Bikol", u"", u"Central Bicolano"),
389
+    (u"en-XA", u"English", u"", u"English")
390
+)

+ 7
- 4
searx/query.py View File

@@ -71,21 +71,24 @@ class RawTextQuery(object):
71 71
                 # check if any language-code is equal with
72 72
                 # declared language-codes
73 73
                 for lc in language_codes:
74
-                    lang_id, lang_name, country = map(str.lower, lc)
74
+                    lang_id, lang_name, country, english_name = map(unicode.lower, lc)
75 75
 
76 76
                     # if correct language-code is found
77 77
                     # set it as new search-language
78 78
                     if lang == lang_id\
79 79
                        or lang_id.startswith(lang)\
80 80
                        or lang == lang_name\
81
+                       or lang == english_name\
81 82
                        or lang.replace('_', ' ') == country:
82 83
                         parse_next = True
83
-                        self.languages.append(lang)
84
-                        break
84
+                        self.languages.append(lang_id)
85
+                        # to ensure best match (first match is not necessarily the best one)
86
+                        if lang == lang_id:
87
+                            break
85 88
 
86 89
             # this force a engine or category
87 90
             if query_part[0] == '!' or query_part[0] == '?':
88
-                prefix = query_part[1:].replace('_', ' ')
91
+                prefix = query_part[1:].replace('-', ' ')
89 92
 
90 93
                 # check if prefix is equal with engine shortcut
91 94
                 if prefix in engine_shortcuts:

+ 3
- 3
searx/templates/courgette/preferences.html View File

@@ -13,9 +13,9 @@
13 13
         <legend>{{ _('Search language') }}</legend>
14 14
         <p>
15 15
             <select name='language'>
16
-                <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
17
-                {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
18
-                <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
16
+                <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
17
+                {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
18
+                <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
19 19
                 {% endfor %}
20 20
             </select>
21 21
         </p>

+ 3
- 3
searx/templates/legacy/preferences.html View File

@@ -14,9 +14,9 @@
14 14
         <legend>{{ _('Search language') }}</legend>
15 15
         <p>
16 16
         <select name='language'>
17
-            <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
18
-            {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
19
-            <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
17
+            <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
18
+            {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
19
+            <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
20 20
             {% endfor %}
21 21
         </select>
22 22
         </p>

+ 1
- 0
searx/templates/oscar/advanced.html View File

@@ -6,4 +6,5 @@
6 6
 <div id="advanced-search-container">
7 7
     {% include 'oscar/categories.html' %}
8 8
     {% include 'oscar/time-range.html' %}
9
+    {% include 'oscar/languages.html' %}
9 10
 </div>

+ 12
- 0
searx/templates/oscar/languages.html View File

@@ -0,0 +1,12 @@
1
+{% if preferences %}
2
+<select class="form-control" name='language'>
3
+{% else %}
4
+<select class="time_range" name='language'>
5
+{% endif %}
6
+	<option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
7
+		{% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
8
+		<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>
9
+			{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}
10
+		</option>
11
+		{% endfor %}
12
+</select>

+ 5
- 6
searx/templates/oscar/preferences.html View File

@@ -40,12 +40,7 @@
40 40
                     {% set language_label = _('Search language') %}
41 41
                     {% set language_info = _('What language do you prefer for search?') %}
42 42
                     {{ preferences_item_header(language_info, language_label, rtl) }}
43
-                        <select class="form-control" name='language'>
44
-                            <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
45
-                            {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
46
-                            <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
47
-                            {% endfor %}
48
-                        </select>
43
+						{% include 'oscar/languages.html' %}
49 44
                     {{ preferences_item_footer(language_info, language_label, rtl) }}
50 45
 
51 46
                     {% set locale_label = _('Interface language') %}
@@ -153,6 +148,7 @@
153 148
 				    <th>{{ _("Allow") }}</th>
154 149
 				    <th>{{ _("Engine name") }}</th>
155 150
 				    <th>{{ _("Shortcut") }}</th>
151
+				    <th>{{ _("Language support") }}</th>
156 152
 				    <th>{{ _("SafeSearch") }}</th>
157 153
 				    <th>{{ _("Time range") }}</th>
158 154
 				    <th>{{ _("Avg. time") }}</th>
@@ -161,6 +157,7 @@
161 157
 				    <th>{{ _("Max time") }}</th>
162 158
 				    <th>{{ _("Avg. time") }}</th>
163 159
 				    <th>{{ _("SafeSearch") }}</th>
160
+				    <th>{{ _("Language support") }}</th>
164 161
 				    <th>{{ _("Shortcut") }}</th>
165 162
 				    <th>{{ _("Engine name") }}</th>
166 163
 				    <th>{{ _("Allow") }}</th>
@@ -175,6 +172,7 @@
175 172
                                     </td>
176 173
                                     <th>{{ search_engine.name }}</th>
177 174
 				    <td>{{ shortcuts[search_engine.name] }}</td>
175
+				    <td><input type="checkbox" {{ "checked" if current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages else ""}} readonly="readonly" disabled="disabled"></td>
178 176
 				    <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td>
179 177
 				    <td><input type="checkbox" {{ "checked" if search_engine.time_range_support==True else ""}} readonly="readonly" disabled="disabled"></td>
180 178
 				    <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>
@@ -183,6 +181,7 @@
183 181
 				    <td class="{{ 'danger' if stats[search_engine.name]['warn_timeout'] else '' }}">{{ search_engine.timeout }}</td>
184 182
 				    <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>
185 183
 				    <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td>
184
+				    <td><input type="checkbox" {{ "checked" if current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages else ""}} readonly="readonly" disabled="disabled"></td>
186 185
 				    <td>{{ shortcuts[search_engine.name] }}</td>
187 186
                                     <th>{{ search_engine.name }}</th>
188 187
                                     <td class="onoff-checkbox">

+ 3
- 3
searx/templates/pix-art/preferences.html View File

@@ -9,9 +9,9 @@
9 9
         <legend>{{ _('Search language') }}</legend>
10 10
         <p>
11 11
         <select name='language'>
12
-            <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Automatic') }}</option>
13
-            {% for lang_id,lang_name,country_name in language_codes | sort(attribute=1) %}
14
-            <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} ({{ country_name }}) - {{ lang_id }}</option>
12
+            <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
13
+            {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
14
+            <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
15 15
             {% endfor %}
16 16
         </select>
17 17
         </p>

+ 8
- 4
searx/webapp.py View File

@@ -330,6 +330,10 @@ def render(template_name, override_theme=None, **kwargs):
330 330
 
331 331
     kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
332 332
 
333
+    kwargs['language_codes'] = language_codes
334
+    if 'current_language' not in kwargs:
335
+        kwargs['current_language'] = request.preferences.get_value('language')
336
+
333 337
     # override url_for function in templates
334 338
     kwargs['url_for'] = url_for_theme
335 339
 
@@ -510,6 +514,7 @@ def index():
510 514
         answers=result_container.answers,
511 515
         infoboxes=result_container.infoboxes,
512 516
         paging=result_container.paging,
517
+        current_language=search.lang,
513 518
         base_url=get_base_url(),
514 519
         theme=get_current_theme_name(),
515 520
         favicons=global_favicons[themes.index(get_current_theme_name())]
@@ -552,7 +557,7 @@ def autocompleter():
552 557
         if not language or language == 'all':
553 558
             language = 'en'
554 559
         else:
555
-            language = language.split('_')[0]
560
+            language = language.split('-')[0]
556 561
         # run autocompletion
557 562
         raw_results.extend(completer(raw_text_query.getSearchQuery(), language))
558 563
 
@@ -615,9 +620,7 @@ def preferences():
615 620
     return render('preferences.html',
616 621
                   locales=settings['locales'],
617 622
                   current_locale=get_locale(),
618
-                  current_language=lang,
619 623
                   image_proxy=image_proxy,
620
-                  language_codes=language_codes,
621 624
                   engines_by_category=categories,
622 625
                   stats=stats,
623 626
                   answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
@@ -627,7 +630,8 @@ def preferences():
627 630
                   themes=themes,
628 631
                   plugins=plugins,
629 632
                   allowed_plugins=allowed_plugins,
630
-                  theme=get_current_theme_name())
633
+                  theme=get_current_theme_name(),
634
+                  preferences=True)
631 635
 
632 636
 
633 637
 @app.route('/image_proxy', methods=['GET'])

+ 99
- 0
utils/update_languages.py View File

@@ -0,0 +1,99 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# This script generates languages.py from
4
+# intersecting each engine's supported languages.
5
+#
6
+# The language's native names are obtained from
7
+# Wikipedia's supported languages.
8
+#
9
+# Output file (languages.py) is written in current directory
10
+# to avoid overwriting in case something goes wrong.
11
+
12
+from requests import get
13
+from re import sub
14
+from lxml.html import fromstring
15
+from json import loads
16
+from sys import path
17
+path.append('../searx')
18
+from searx.engines import engines
19
+
20
+# list of language names
21
+wiki_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'
22
+google_languages_url = 'https://www.google.com/preferences?#languages'
23
+
24
+google_json_name = 'google.preferences.langMap'
25
+
26
+languages = {}
27
+
28
+# Get language names from Wikipedia.
29
+def get_wikipedia_languages():
30
+    response = get(wiki_languages_url)
31
+    dom = fromstring(response.text)
32
+    tables = dom.xpath('//table[contains(@class,"sortable")]')
33
+    for table in tables:
34
+        # exclude header row
35
+        trs = table.xpath('.//tr')[1:]
36
+        for tr in trs:
37
+            td = tr.xpath('./td')
38
+            code = td[3].xpath('./a')[0].text
39
+            name = td[2].xpath('./a')[0].text
40
+            english_name = td[1].xpath('./a')[0].text
41
+            
42
+            if code not in languages:
43
+                languages[code] = (name, '', english_name)
44
+
45
+# Get language names from Google.
46
+def get_google_languages():
47
+    response = get(google_languages_url)
48
+    dom = fromstring(response.text)
49
+    options = dom.xpath('//select[@name="hl"]/option')
50
+    for option in options:
51
+        code = option.xpath('./@value')[0]
52
+        name = option.text[:-1]
53
+
54
+        if code not in languages:
55
+            languages[code] = (name, '', '')
56
+
57
+# Join all language lists.
58
+# iterate all languages supported by each engine
59
+def join_language_lists():
60
+    for engine_name in engines:
61
+        for locale in engines[engine_name].supported_languages:
62
+            locale = locale.replace('_', '-')
63
+            if locale not in languages:
64
+                # try to get language name
65
+                language = languages.get(locale.split('-')[0], None)
66
+                if language == None:
67
+                    print engine_name + ": " + locale
68
+                    continue
69
+
70
+                (name, country, english) = language
71
+                languages[locale] = (name, country, english)
72
+
73
+# Write languages.py.
74
+def write_languages_file():
75
+    new_file = open('languages.py', 'w')
76
+    file_content = '# -*- coding: utf-8 -*-\n'
77
+    file_content += '# list of language codes\n'
78
+    file_content += '# this file is generated automatically by utils/update_search_languages.py\n'
79
+    file_content += '\nlanguage_codes = ('
80
+    for code in languages:
81
+        (name, country, english) = languages[code]
82
+        file_content += '\n    (u"' + code + '"'\
83
+                        + ', u"' + name + '"'\
84
+                        + ', u"' + country[1:-1] + '"'\
85
+                        + ', u"' + english + '"),'
86
+    # remove last comma
87
+    file_content = file_content[:-1]
88
+    file_content += '\n)\n'
89
+    new_file.write(file_content.encode('utf8'))
90
+    new_file.close()
91
+
92
+def main():
93
+    get_wikipedia_languages()
94
+    get_google_languages()
95
+    join_language_lists()
96
+    write_languages_file()
97
+
98
+if __name__ == "__main__":
99
+    main()