Browse Source

update engines_languages.json and languages.py

Also, fix fetch_languages.py so it can run on python3.
marc 7 years ago
parent
commit
44085e31d0

+ 1
- 1
searx/data/engines_languages.json
File diff suppressed because it is too large
View File


+ 1
- 1
searx/engines/duckduckgo.py View File

@@ -134,4 +134,4 @@ def _fetch_supported_languages(resp):
134 134
     regions_json = loads(response_page)
135 135
     supported_languages = map((lambda x: x[3:] + '-' + x[:2].upper()), regions_json.keys())
136 136
 
137
-    return supported_languages
137
+    return list(supported_languages)

+ 1
- 1
searx/engines/swisscows.py View File

@@ -118,7 +118,7 @@ def _fetch_supported_languages(resp):
118 118
     dom = fromstring(resp.text)
119 119
     options = dom.xpath('//div[@id="regions-popup"]//ul/li/a')
120 120
     for option in options:
121
-        code = option.xpath('./@data-val')[0]
121
+        code = option.xpath('./@data-search-language')[0]
122 122
         if code.startswith('nb-'):
123 123
             code = code.replace('nb', 'no', 1)
124 124
         supported_languages.append(code)

+ 8
- 10
searx/languages.py View File

@@ -5,6 +5,11 @@
5 5
 language_codes = (
6 6
     (u"ar-SA", u"العربية", u"", u"Arabic"),
7 7
     (u"bg-BG", u"Български", u"", u"Bulgarian"),
8
+    (u"ca", u"Català", u"", u"Catalan"),
9
+    (u"ca-AD", u"Català", u"Andorra", u"Catalan"),
10
+    (u"ca-CT", u"Català", u"", u"Catalan"),
11
+    (u"ca-ES", u"Català", u"Espanya", u"Catalan"),
12
+    (u"ca-FR", u"Català", u"França", u"Catalan"),
8 13
     (u"cs-CZ", u"Čeština", u"", u"Czech"),
9 14
     (u"da-DK", u"Dansk", u"", u"Danish"),
10 15
     (u"de", u"Deutsch", u"", u"German"),
@@ -15,9 +20,7 @@ language_codes = (
15 20
     (u"en", u"English", u"", u"English"),
16 21
     (u"en-AU", u"English", u"Australia", u"English"),
17 22
     (u"en-CA", u"English", u"Canada", u"English"),
18
-    (u"en-CY", u"English", u"Cyprus", u"English"),
19 23
     (u"en-GB", u"English", u"United Kingdom", u"English"),
20
-    (u"en-GD", u"English", u"Grenada", u"English"),
21 24
     (u"en-ID", u"English", u"Indonesia", u"English"),
22 25
     (u"en-IE", u"English", u"Ireland", u"English"),
23 26
     (u"en-IN", u"English", u"India", u"English"),
@@ -28,6 +31,7 @@ language_codes = (
28 31
     (u"en-US", u"English", u"United States", u"English"),
29 32
     (u"en-ZA", u"English", u"South Africa", u"English"),
30 33
     (u"es", u"Español", u"", u"Spanish"),
34
+    (u"es-AD", u"Español", u"Andorra", u"Spanish"),
31 35
     (u"es-AR", u"Español", u"Argentina", u"Spanish"),
32 36
     (u"es-CL", u"Español", u"Chile", u"Spanish"),
33 37
     (u"es-CO", u"Español", u"Colombia", u"Spanish"),
@@ -38,38 +42,32 @@ language_codes = (
38 42
     (u"et-EE", u"Eesti", u"", u"Estonian"),
39 43
     (u"fi-FI", u"Suomi", u"", u"Finnish"),
40 44
     (u"fr", u"Français", u"", u"French"),
45
+    (u"fr-AD", u"Français", u"Andorre", u"French"),
41 46
     (u"fr-BE", u"Français", u"Belgique", u"French"),
42 47
     (u"fr-CA", u"Français", u"Canada", u"French"),
43 48
     (u"fr-CH", u"Français", u"Suisse", u"French"),
44 49
     (u"fr-FR", u"Français", u"France", u"French"),
45 50
     (u"he-IL", u"עברית", u"", u"Hebrew"),
46
-    (u"hr-HR", u"Hrvatski", u"", u"Croatian"),
47 51
     (u"hu-HU", u"Magyar", u"", u"Hungarian"),
48
-    (u"id-ID", u"Bahasa Indonesia", u"", u"Indonesian"),
49 52
     (u"it", u"Italiano", u"", u"Italian"),
50 53
     (u"it-CH", u"Italiano", u"Svizzera", u"Italian"),
51 54
     (u"it-IT", u"Italiano", u"Italia", u"Italian"),
52 55
     (u"ja-JP", u"日本語", u"", u"Japanese"),
53 56
     (u"ko-KR", u"한국어", u"", u"Korean"),
54
-    (u"lt-LT", u"Lietuvių", u"", u"Lithuanian"),
55
-    (u"lv-LV", u"Latviešu", u"", u"Latvian"),
56
-    (u"ms-MY", u"Bahasa Melayu", u"", u"Malay"),
57 57
     (u"nl", u"Nederlands", u"", u"Dutch"),
58 58
     (u"nl-BE", u"Nederlands", u"België", u"Dutch"),
59 59
     (u"nl-NL", u"Nederlands", u"Nederland", u"Dutch"),
60 60
     (u"no-NO", u"Norsk", u"", u"Norwegian"),
61 61
     (u"pl-PL", u"Polski", u"", u"Polish"),
62 62
     (u"pt", u"Português", u"", u"Portuguese"),
63
+    (u"pt-AD", u"Português", u"Andorra", u"Portuguese"),
63 64
     (u"pt-BR", u"Português", u"Brasil", u"Portuguese"),
64 65
     (u"pt-PT", u"Português", u"Portugal", u"Portuguese"),
65 66
     (u"ro-RO", u"Română", u"", u"Romanian"),
66 67
     (u"ru-RU", u"Русский", u"", u"Russian"),
67
-    (u"sk-SK", u"Slovenčina", u"", u"Slovak"),
68
-    (u"sl", u"Slovenščina", u"", u"Slovenian"),
69 68
     (u"sv-SE", u"Svenska", u"", u"Swedish"),
70 69
     (u"th-TH", u"ไทย", u"", u"Thai"),
71 70
     (u"tr-TR", u"Türkçe", u"", u"Turkish"),
72
-    (u"vi-VN", u"Tiếng Việt", u"", u"Vietnamese"),
73 71
     (u"zh", u"中文", u"", u"Chinese"),
74 72
     (u"zh-CN", u"中文", u"中国", u"Chinese"),
75 73
     (u"zh-HK", u"中文", u"香港", u"Chinese"),

+ 3
- 3
tests/unit/engines/test_swisscows.py View File

@@ -139,9 +139,9 @@ class TestSwisscowsEngine(SearxTestCase):
139 139
             <div id="regions-popup">
140 140
                 <div>
141 141
                     <ul>
142
-                        <li><a data-val="browser"></a></li>
143
-                        <li><a data-val="de-CH"></a></li>
144
-                        <li><a data-val="fr-CH"></a></li>
142
+                        <li><a data-search-language="browser"></a></li>
143
+                        <li><a data-search-language="de-CH"></a></li>
144
+                        <li><a data-search-language="fr-CH"></a></li>
145 145
                     </ul>
146 146
                 </div>
147 147
             </div>

+ 8
- 8
utils/fetch_languages.py View File

@@ -8,13 +8,13 @@
8 8
 # are written in current directory to avoid overwriting in case something goes wrong.
9 9
 
10 10
 from requests import get
11
-from urllib import urlencode
12 11
 from lxml.html import fromstring
13
-from json import loads, dumps
12
+from json import loads, dump
14 13
 import io
15 14
 from sys import path
16 15
 path.append('../searx')  # noqa
17 16
 from searx import settings
17
+from searx.url_utils import urlencode
18 18
 from searx.engines import initialize_engines, engines
19 19
 
20 20
 # Geonames API for country names.
@@ -70,7 +70,7 @@ def get_country_name(locale):
70 70
     json = loads(response.text)
71 71
     content = json.get('geonames', None)
72 72
     if content is None or len(content) != 1:
73
-        print "No country name found for " + locale[0] + "-" + locale[1]
73
+        print("No country name found for " + locale[0] + "-" + locale[1])
74 74
         return ''
75 75
 
76 76
     return content[0].get('countryName', '')
@@ -84,11 +84,11 @@ def fetch_supported_languages():
84 84
             try:
85 85
                 engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
86 86
             except Exception as e:
87
-                print e
87
+                print(e)
88 88
 
89 89
     # write json file
90 90
     with io.open(engines_languages_file, "w", encoding="utf-8") as f:
91
-        f.write(unicode(dumps(engines_languages, ensure_ascii=False, encoding="utf-8")))
91
+        dump(engines_languages, f, ensure_ascii=False)
92 92
 
93 93
 
94 94
 # Join all language lists.
@@ -97,7 +97,7 @@ def join_language_lists():
97 97
     global languages
98 98
     # include wikipedia first for more accurate language names
99 99
     languages = {code: lang for code, lang
100
-                 in engines_languages['wikipedia'].iteritems()
100
+                 in engines_languages['wikipedia'].items()
101 101
                  if valid_code(code)}
102 102
 
103 103
     for engine_name in engines_languages:
@@ -121,7 +121,7 @@ def join_language_lists():
121 121
     # filter list to include only languages supported by most engines
122 122
     min_supported_engines = int(0.70 * len(engines_languages))
123 123
     languages = {code: lang for code, lang
124
-                 in languages.iteritems()
124
+                 in languages.items()
125 125
                  if len(lang.get('counter', [])) >= min_supported_engines or
126 126
                  len(languages.get(code.split('-')[0], {}).get('counter', [])) >= min_supported_engines}
127 127
 
@@ -165,7 +165,7 @@ def filter_single_country_languages():
165 165
 
166 166
 # Write languages.py.
167 167
 def write_languages_file():
168
-    new_file = open(languages_file, 'w')
168
+    new_file = open(languages_file, 'wb')
169 169
     file_content = '# -*- coding: utf-8 -*-\n'\
170 170
                    + '# list of language codes\n'\
171 171
                    + '# this file is generated automatically by utils/update_search_languages.py\n'\