summaryrefslogtreecommitdiff
path: root/utils
diff options
context:
space:
mode:
authorJohannes 'josch' Schauer <josch@debian.org>2019-01-06 22:42:31 +0100
committerJohannes 'josch' Schauer <josch@debian.org>2019-01-06 23:42:19 +0100
commited48940eb3d6c12d9800bf7b490530d80c969e83 (patch)
treed8cb3d1506475576e85e5351a1e9b4e752326059 /utils
parent728a7e0706c2d867a5de7247ea8c4c5f27bee346 (diff)
New upstream version 0.15.0+dfsg1
Diffstat (limited to 'utils')
-rwxr-xr-xutils/fetch_firefox_version.py73
-rw-r--r--utils/fetch_languages.py266
2 files changed, 207 insertions, 132 deletions
diff --git a/utils/fetch_firefox_version.py b/utils/fetch_firefox_version.py
new file mode 100755
index 0000000..ed17958
--- /dev/null
+++ b/utils/fetch_firefox_version.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+# set path
+from sys import path
+from os.path import realpath, dirname, join
+path.append(realpath(dirname(realpath(__file__)) + '/../'))
+
+#
+import json
+import requests
+import re
+from distutils.version import LooseVersion, StrictVersion
+from lxml import html
+from searx.url_utils import urlparse, urljoin
+from searx import searx_dir
+
+URL = 'https://ftp.mozilla.org/pub/firefox/releases/'
+RELEASE_PATH = '/pub/firefox/releases/'
+
+NORMAL_REGEX = re.compile('^[0-9]+\.[0-9](\.[0-9])?$')
+# BETA_REGEX = re.compile('.*[0-9]b([0-9\-a-z]+)$')
+# ESR_REGEX = re.compile('^[0-9]+\.[0-9](\.[0-9])?esr$')
+
+#
+useragents = {
+ "versions": (),
+ "os": ('Windows NT 10; WOW64',
+ 'X11; Linux x86_64'),
+ "ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}"
+}
+
+
+def fetch_firefox_versions():
+ resp = requests.get(URL, timeout=2.0)
+ if resp.status_code != 200:
+ raise Exception("Error fetching firefox versions, HTTP code " + resp.status_code)
+ else:
+ dom = html.fromstring(resp.text)
+ versions = []
+
+ for link in dom.xpath('//a/@href'):
+ url = urlparse(urljoin(URL, link))
+ path = url.path
+ if path.startswith(RELEASE_PATH):
+ version = path[len(RELEASE_PATH):-1]
+ if NORMAL_REGEX.match(version):
+ versions.append(LooseVersion(version))
+
+ list.sort(versions, reverse=True)
+ return versions
+
+
+def fetch_firefox_last_versions():
+ versions = fetch_firefox_versions()
+
+ result = []
+ major_last = versions[0].version[0]
+ major_list = (major_last, major_last - 1)
+ for version in versions:
+ major_current = version.version[0]
+ if major_current in major_list:
+ result.append(version.vstring)
+
+ return result
+
+
+def get_useragents_filename():
+ return join(join(searx_dir, "data"), "useragents.json")
+
+
+useragents["versions"] = fetch_firefox_last_versions()
+with open(get_useragents_filename(), "w") as f:
+ json.dump(useragents, f, indent=4, ensure_ascii=False)
diff --git a/utils/fetch_languages.py b/utils/fetch_languages.py
index 424f400..a6af073 100644
--- a/utils/fetch_languages.py
+++ b/utils/fetch_languages.py
@@ -2,169 +2,169 @@
# This script generates languages.py from intersecting each engine's supported languages.
#
-# The country names are obtained from http://api.geonames.org which requires registering as a user.
-#
# Output files (engines_languages.json and languages.py)
# are written in current directory to avoid overwriting in case something goes wrong.
-from requests import get
-from lxml.html import fromstring
-from json import loads, dump
+from json import dump
import io
from sys import path
+from babel import Locale, UnknownLocaleError
+from babel.languages import get_global
+
path.append('../searx') # noqa
from searx import settings
-from searx.url_utils import urlencode
from searx.engines import initialize_engines, engines
-# Geonames API for country names.
-geonames_user = '' # ADD USER NAME HERE
-country_names_url = 'http://api.geonames.org/countryInfoJSON?{parameters}'
-
# Output files.
engines_languages_file = 'engines_languages.json'
languages_file = 'languages.py'
-engines_languages = {}
-
-
-# To filter out invalid codes and dialects.
-def valid_code(lang_code):
- # filter invalid codes
- # sl-SL is technically not invalid, but still a mistake
- invalid_codes = ['sl-SL', 'wt-WT', 'jw']
- invalid_countries = ['UK', 'XA', 'XL']
- if lang_code[:2] == 'xx'\
- or lang_code in invalid_codes\
- or lang_code[-2:] in invalid_countries\
- or is_dialect(lang_code):
- return False
-
- return True
-
-
-# Language codes with any additional tags other than language and country.
-def is_dialect(lang_code):
- lang_code = lang_code.split('-')
- if len(lang_code) > 2 or len(lang_code[0]) > 3:
- return True
- if len(lang_code) == 2 and len(lang_code[1]) > 2:
- return True
-
- return False
-
-
-# Get country name in specified language.
-def get_country_name(locale):
- if geonames_user is '':
- return ''
-
- locale = locale.split('-')
- if len(locale) != 2:
- return ''
-
- url = country_names_url.format(parameters=urlencode({'lang': locale[0],
- 'country': locale[1],
- 'username': geonames_user}))
- response = get(url)
- json = loads(response.text)
- content = json.get('geonames', None)
- if content is None or len(content) != 1:
- print("No country name found for " + locale[0] + "-" + locale[1])
- return ''
-
- return content[0].get('countryName', '')
-
# Fetchs supported languages for each engine and writes json file with those.
def fetch_supported_languages():
- initialize_engines(settings['engines'])
+ engines_languages = {}
for engine_name in engines:
if hasattr(engines[engine_name], 'fetch_supported_languages'):
try:
engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
+ if type(engines_languages[engine_name]) == list:
+ engines_languages[engine_name] = sorted(engines_languages[engine_name])
except Exception as e:
print(e)
# write json file
with io.open(engines_languages_file, "w", encoding="utf-8") as f:
- dump(engines_languages, f, ensure_ascii=False)
+ dump(engines_languages, f, ensure_ascii=False, indent=4, separators=(',', ': '))
+ return engines_languages
+
+
+# Get babel Locale object from lang_code if possible.
+def get_locale(lang_code):
+ try:
+ locale = Locale.parse(lang_code, sep='-')
+ return locale
+ except (UnknownLocaleError, ValueError):
+ return None
+
+
+# Append engine_name to list of engines that support locale.
+def add_engine_counter(lang_code, engine_name, languages):
+ if lang_code in languages:
+ if 'counter' not in languages[lang_code]:
+ languages[lang_code]['counter'] = [engine_name]
+ elif engine_name not in languages[lang_code]['counter']:
+ languages[lang_code]['counter'].append(engine_name)
-# Join all language lists.
-# Iterate all languages supported by each engine.
-def join_language_lists():
- global languages
- # include wikipedia first for more accurate language names
- languages = {code: lang for code, lang
- in engines_languages['wikipedia'].items()
- if valid_code(code)}
+# Join all language lists.
+# TODO: Add language names from engine's language list if name not known by babel.
+def join_language_lists(engines_languages):
+ language_list = {}
for engine_name in engines_languages:
- for locale in engines_languages[engine_name]:
- if valid_code(locale):
- # if language is not on list or if it has no name yet
- if locale not in languages or not languages[locale].get('name'):
- if isinstance(engines_languages[engine_name], dict):
- languages[locale] = engines_languages[engine_name][locale]
- else:
- languages[locale] = {}
-
- # add to counter of engines that support given language
- lang = locale.split('-')[0]
- if lang in languages:
- if 'counter' not in languages[lang]:
- languages[lang]['counter'] = [engine_name]
- elif engine_name not in languages[lang]['counter']:
- languages[lang]['counter'].append(engine_name)
-
- # filter list to include only languages supported by most engines
- min_supported_engines = int(0.70 * len(engines_languages))
- languages = {code: lang for code, lang
- in languages.items()
- if len(lang.get('counter', [])) >= min_supported_engines or
- len(languages.get(code.split('-')[0], {}).get('counter', [])) >= min_supported_engines}
-
- # get locales that have no name or country yet
- for locale in languages.keys():
- # try to get language names
- if not languages[locale].get('name'):
- name = languages.get(locale.split('-')[0], {}).get('name', None)
- if name:
- languages[locale]['name'] = name
- else:
- # filter out locales with no name
- del languages[locale]
- continue
-
- # try to get language name in english
- if not languages[locale].get('english_name'):
- languages[locale]['english_name'] = languages.get(locale.split('-')[0], {}).get('english_name', '')
-
- # try to get country name
- if locale.find('-') > 0 and not languages[locale].get('country'):
- languages[locale]['country'] = get_country_name(locale) or ''
-
-
-# Remove countryless language if language is featured in only one country.
-def filter_single_country_languages():
- prev_lang = None
- prev_code = None
- for code in sorted(languages):
- lang = code.split('-')[0]
- if lang == prev_lang:
+ for lang_code in engines_languages[engine_name]:
+
+ # apply custom fixes if necessary
+ if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():
+ lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()
+ if lang_code == alias)
+
+ locale = get_locale(lang_code)
+
+ # ensure that lang_code uses standard language and country codes
+ if locale and locale.territory:
+ lang_code = locale.language + '-' + locale.territory
+
+ # add locale if it's not in list
+ if lang_code not in language_list:
+ if locale:
+ language_list[lang_code] = {'name': locale.get_language_name().title(),
+ 'english_name': locale.english_name,
+ 'country': locale.get_territory_name() or ''}
+
+ # also add language without country
+ if locale.language not in language_list:
+ language_list[locale.language] = {'name': locale.get_language_name().title(),
+ 'english_name': locale.english_name}
+ else:
+ language_list[lang_code] = {}
+
+ # count engine for both language_country combination and language alone
+ add_engine_counter(lang_code, engine_name, language_list)
+ add_engine_counter(lang_code.split('-')[0], engine_name, language_list)
+
+ return language_list
+
+
+# Filter language list so it only includes the most supported languages and countries.
+def filter_language_list(all_languages):
+ min_supported_engines = 10
+ main_engines = [engine_name for engine_name in engines.keys()
+ if 'general' in engines[engine_name].categories and
+ engines[engine_name].supported_languages and
+ not engines[engine_name].disabled]
+
+ # filter list to include only languages supported by most engines or all default general engines
+ filtered_languages = {code: lang for code, lang
+ in all_languages.items()
+ if (len(lang.get('counter', [])) >= min_supported_engines or
+ all(main_engine in lang.get('counter', [])
+ for main_engine in main_engines))}
+
+ return filtered_languages
+
+
+# Add country codes to languages without one and filter out language codes.
+def assign_country_codes(filtered_languages, all_languages):
+ sorted_languages = sorted(all_languages,
+ key=lambda lang: len(all_languages[lang].get('counter', [])),
+ reverse=True)
+ previous_lang = None
+ previous_code = None
+ countries = 0
+ for current_code in sorted(filtered_languages):
+ current_lang = current_code.split('-')[0]
+
+ # count country codes per language
+ if current_lang == previous_lang:
countries += 1
+
else:
- if prev_lang is not None and countries == 1:
- del languages[prev_lang]
- languages[prev_code]['country'] = ''
+ if previous_lang is not None:
+ # if language has no single country code
+ if countries == 0:
+ # try to get country code with most supported engines
+ for l in sorted_languages:
+ l_parts = l.split('-')
+ if len(l_parts) == 2 and l_parts[0] == previous_lang:
+ filtered_languages[l] = all_languages[l]
+ filtered_languages[l]['country'] = ''
+ countries = 1
+ break
+
+ if countries == 0:
+ # get most likely country code from babel
+ subtags = get_global('likely_subtags').get(previous_lang)
+ if subtags:
+ subtag_parts = subtags.split('_')
+ new_code = subtag_parts[0] + '-' + subtag_parts[-1]
+ filtered_languages[new_code] = all_languages[previous_lang]
+ countries = 1
+
+ if countries == 1:
+ # remove countryless version of language if there's only one country
+ del filtered_languages[previous_lang]
+ if previous_code in filtered_languages:
+ filtered_languages[previous_code]['country'] = ''
+
countries = 0
- prev_lang = lang
- prev_code = code
+ previous_lang = current_lang
+
+ previous_code = current_code
# Write languages.py.
-def write_languages_file():
+def write_languages_file(languages):
new_file = open(languages_file, 'wb')
file_content = '# -*- coding: utf-8 -*-\n'\
+ '# list of language codes\n'\
@@ -183,7 +183,9 @@ def write_languages_file():
if __name__ == "__main__":
- fetch_supported_languages()
- join_language_lists()
- filter_single_country_languages()
- write_languages_file()
+ initialize_engines(settings['engines'])
+ engines_languages = fetch_supported_languages()
+ all_languages = join_language_lists(engines_languages)
+ filtered_languages = filter_language_list(all_languages)
+ assign_country_codes(filtered_languages, all_languages)
+ write_languages_file(filtered_languages)