- search.engines_path : path of engines.yml

- search.include_engines_path : add engines definition (doesn't replace the content of engines.yml)

doi.yml, engines.yml, locales.yml are loaded directly into doi, engines, locales (not more prefix)
some bug fixes on oa_doi_rewrite

searx/__init__.py

@@ -36,7 +36,7 @@ def build_key_to_index(seq, key):
     return dict((d[key], index) for (index, d) in enumerate(seq))
 
 
-def check_file(file_name):
+def file_or_none(file_name):
     if isfile(file_name):
         return file_name
     else:
@@ -48,22 +48,41 @@ def load_yaml(file_name):
         return load(file_yaml)
 
 
-def load_embedded_yaml(name):
-    file_name = join(searx_dir, name)
-    logger.debug('read configuration from %s', file_name)
-    if check_file(file_name):
-        return load_yaml(file_name)
+def get_embedded_filename(name):
+    return join(searx_dir, name)
+
+
+def update_settings_with_path(settings, path, key=None):
+    if path is None or path == '':
+        return
+    elif isfile(path):
+        logger.debug('read configuration from %s', path)
+        y = load_yaml(path)
+        if isinstance(settings, dict):
+            settings.update(y)
+        elif isinstance(settings, list):
+            kindex = build_key_to_index(settings, key)
+            for e in y:
+                if key is not None and key in e:
+                    i = kindex.get(e[key], None)
+                    if i is not None:
+                        settings[i] = e
+                        continue
+                settings.append(e)
+        else:
+            logger.error('%s content is neither a list nor a dictionary')
     else:
-        logger.warning('{0} is not found.')
+        logger.error('%s is not a file', path)
+
 
 # find location of settings.yml
 if 'SEARX_SETTINGS_PATH' in environ:
     # if possible set path to settings using the
     # enviroment variable SEARX_SETTINGS_PATH
-    user_settings_path = check_file(environ['SEARX_SETTINGS_PATH'])
+    user_settings_path = file_or_none(environ['SEARX_SETTINGS_PATH'])
 else:
     # if not, get it from searx code base or last solution from /etc/searx
-    user_settings_path = check_file(join(searx_dir, 'settings.yml')) or check_file('/etc/searx/settings.yml')
+    user_settings_path = file_or_none(get_embedded_filename('settings.yml')) or file_or_none('/etc/searx/settings.yml')
 
 if not user_settings_path:
     raise Exception('settings.yml not found')
@@ -106,15 +125,26 @@ if OPENSSL_VERSION_INFO[0:3] < (1, 0, 2):
 '''
 Load all settings
 '''
-
 # settings are merged from different yml files
-settings = dict()
+settings = {
+    'search': {
+        'engines': list()
+    },
+    'engines': list(),
+    'locales': dict(),
+    'doi': dict()
+}
 # load embedded settings first
-settings.update(load_embedded_yaml('engines.yml'))
-settings.update(load_embedded_yaml('doi.yml'))
-settings.update(load_embedded_yaml('locales.yml'))
-# load user settings at the end (may override embedded settings)
+update_settings_with_path(settings['engines'],
+                          user_settings.get('search', {}).get('engines_path', None)
+                          or get_embedded_filename('engines.yml'))
+update_settings_with_path(settings['doi'], get_embedded_filename('doi.yml'))
+update_settings_with_path(settings['locales'], get_embedded_filename('locales.yml'))
+# load user settings (may override embedded settings)
 settings.update(user_settings)
+# add additional engines (replace engine definition if it already exists)
+update_settings_with_path(settings['engines'],
+                          user_settings.get('search', {}).get('include_engines_path', None), 'name')
 # are there some user engine settings ?
 user_engine_settings = settings.get('search', {}).get('engines', None)
 if user_engine_settings:

searx/doi.yml

@@ -1,6 +1,5 @@
-doi_resolvers :
+resolvers :
   oadoi.org : 'https://oadoi.org/'
   doi.org : 'https://doi.org/'
   doai.io  : 'http://doai.io/'
-
-default_doi_resolver : 'oadoi.org'
+default: 'oadoi.org'

searx/locales.yml

@@ -1,32 +1,31 @@
-locales:
-    en : English
-    ar : العَرَبِيَّة (Arabic)
-    bg : Български (Bulgarian)
-    cs : Čeština (Czech)
-    da : Dansk (Danish)
-    de : Deutsch (German)
-    el_GR : Ελληνικά (Greek_Greece)
-    eo : Esperanto (Esperanto)
-    es : Español (Spanish)
-    fi : Suomi (Finnish)
-    fil : Wikang Filipino (Filipino)
-    fr : Français (French)
-    he : עברית (Hebrew)
-    hr : Hrvatski (Croatian)
-    hu : Magyar (Hungarian)
-    it : Italiano (Italian)
-    ja : 日本語 (Japanese)
-    nl : Nederlands (Dutch)
-    pl : Polszczyzna (Polish)
-    pt : Português (Portuguese)
-    pt_BR : Português (Portuguese_Brazil)
-    ro : Română (Romanian)
-    ru : Русский (Russian)
-    sk : Slovenčina (Slovak)
-    sl : Slovenski (Slovene)
-    sr : српски (Serbian)
-    sv : Svenska (Swedish)
-    tr : Türkçe (Turkish)
-    uk : українська мова (Ukrainian)
-    zh : 中文 (Chinese)
-    zh_TW : 國語 (Taiwanese Mandarin)
+  en : English
+  ar : العَرَبِيَّة (Arabic)
+  bg : Български (Bulgarian)
+  cs : Čeština (Czech)
+  da : Dansk (Danish)
+  de : Deutsch (German)
+  el_GR : Ελληνικά (Greek_Greece)
+  eo : Esperanto (Esperanto)
+  es : Español (Spanish)
+  fi : Suomi (Finnish)
+  fil : Wikang Filipino (Filipino)
+  fr : Français (French)
+  he : עברית (Hebrew)
+  hr : Hrvatski (Croatian)
+  hu : Magyar (Hungarian)
+  it : Italiano (Italian)
+  ja : 日本語 (Japanese)
+  nl : Nederlands (Dutch)
+  pl : Polszczyzna (Polish)
+  pt : Português (Portuguese)
+  pt_BR : Português (Portuguese_Brazil)
+  ro : Română (Romanian)
+  ru : Русский (Russian)
+  sk : Slovenčina (Slovak)
+  sl : Slovenski (Slovene)
+  sr : српски (Serbian)
+  sv : Svenska (Swedish)
+  tr : Türkçe (Turkish)
+  uk : українська мова (Ukrainian)
+  zh : 中文 (Chinese)
+  zh_TW : 國語 (Taiwanese Mandarin)

searx/plugins/oa_doi_rewrite.py

@@ -11,7 +11,7 @@ description = gettext('Avoid paywalls by redirecting to open-access versions of
 default_on = False
 preference_section = 'privacy'
 
-doi_resolvers = settings['doi_resolvers']
+doi_resolvers = settings['doi']['resolvers']
 
 
 def extract_doi(url):
@@ -26,19 +26,20 @@ def extract_doi(url):
 
 
 def get_doi_resolver(args, preference_doi_resolver):
-    doi_resolvers = settings['doi_resolvers']
+    doi_resolvers = settings['doi']['resolvers']
     doi_resolver = args.get('doi_resolver', preference_doi_resolver)[0]
     if doi_resolver not in doi_resolvers:
-        doi_resolvers = settings['default_doi_resolver']
+        doi_resolver = settings['doi']['default']
     return doi_resolver
 
 
 def on_result(request, search, result):
     doi = extract_doi(result['parsed_url'])
     if doi and len(doi) < 50:
-        for suffix in ('/', '.pdf', '/full', '/meta', '/abstract'):
+        for suffix in ('/', '.pdf', '.full', '/pdf', '/full', '/meta', '/abstract'):
             if doi.endswith(suffix):
                 doi = doi[:-len(suffix)]
-        result['url'] = get_doi_resolver(request.args, request.preferences.get_value('doi_resolver')) + doi
+        base_url = doi_resolvers[get_doi_resolver(request.args, request.preferences.get_value('doi_resolver'))]
+        result['url'] = base_url + doi
         result['parsed_url'] = urlparse(result['url'])
     return True

searx/preferences.py

@@ -14,7 +14,7 @@ COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5  # 5 years
 LANGUAGE_CODES = [l[0] for l in languages]
 DISABLED = 0
 ENABLED = 1
-DOI_RESOLVERS = list(settings['doi_resolvers'])
+DOI_RESOLVERS = list(settings['doi']['resolvers'])
 
 
 class MissingArgumentException(Exception):

searx/settings.yml

@@ -6,7 +6,9 @@ search:
     safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict
     autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
     language : "en-US"
-    engines:
+    engines_path: "" # engines.yml path - leave it blank if you don't change
+    include_engines_path: "" # user defined engines file path - leave it blank if you don't change
+    engines: # enabled engines with additional settings (timeout for example)
       - name: arch linux wiki
       #- name: archive is
       - name: arxiv

searx/webapp.py

@@ -716,7 +716,7 @@ def preferences():
                   shortcuts={y: x for x, y in engine_shortcuts.items()},
                   themes=themes,
                   plugins=plugins,
-                  doi_resolvers=settings['doi_resolvers'],
+                  doi_resolvers=settings['doi']['resolvers'],
                   current_doi_resolver=get_doi_resolver(request.args, request.preferences.get_value('doi_resolver')),
                   allowed_plugins=allowed_plugins,
                   theme=get_current_theme_name(),
@@ -863,8 +863,8 @@ def config():
                     'safe_search': settings['search']['safe_search'],
                     'default_theme': settings['ui']['default_theme'],
                     'version': VERSION_STRING,
-                    'doi_resolvers': [r for r in settings['doi_resolvers']],
-                    'default_doi_resolver': settings['default_doi_resolver'],
+                    'doi_resolvers': [r for r in settings['doi']['resolvers']],
+                    'default_doi_resolver': settings['doi']['default'],
                     })