jenkins-bot submitted this change.
[bugfix] Lookup the code parameter in xdict first
i18n.py:
- Lookup the code parameter in xdict first (T255917)
- If code is not in xdict but the family name is found,
use xdict[family.name] to find a localization
- If fallback is False and no localization is found for a given family
always use 'wikipedia' entry if present even if there is a family entry
given. This family fallback prevents code duplication within xdict.
- The fallback lookup with 'wikipedia' key is made recursively with
only the xdict['wikipedia'] branch as actual parameter
- do not return directly if the fallback is false and no key is found.
This ensures the 'wikipedia' fallback is done
- update and extend translate method docstring
- update and extend package docstring
- ignore __iter__ and __len__ of _PluralMappingAliasfrom coverage
i18n_tests.py:
- rename TestTranslate to TestFallbackTranslate because these tests
have fallback enabled
- remove TestFallbackTranslate.setUp and use class attributes for
test dicts
- add a new TestTranslate test class to test extended xdict
- add a fake Site object which just holds code and family.name
attributes because the usage of pywikibot.Site is disabled by
DisableSiteMixin
Bug: T255917
Change-Id: I9bb6688064db7ac0050820889f78845aee4308d6
---
M pywikibot/i18n.py
M tests/i18n_tests.py
2 files changed, 206 insertions(+), 58 deletions(-)
diff --git a/pywikibot/i18n.py b/pywikibot/i18n.py
index 6e89680..682c1d9 100644
--- a/pywikibot/i18n.py
+++ b/pywikibot/i18n.py
@@ -1,20 +1,19 @@
# -*- coding: utf-8 -*-
-"""
-Various i18n functions.
+"""Various i18n functions.
-Helper functions for both the internal translation system
-and for TranslateWiki-based translations.
+Helper functions for both the internal localization system and for
+TranslateWiki-based translations.
By default messages are assumed to reside in a package called
-'scripts.i18n'. In pywikibot 3+, that package is not packaged
-with pywikibot, and pywikibot 3+ does not have a hard dependency
-on any i18n messages. However, there are three user input questions
-in pagegenerators which will use i18n messages if they can be loaded.
+'scripts.i18n'. In pywikibot 3+, that package is not packaged with
+pywikibot, and pywikibot 3+ does not have a hard dependency on any i18n
+messages. However, there are three user input questions in pagegenerators
+which will use i18n messages if they can be loaded.
The default message location may be changed by calling
-L{set_message_package} with a package name. The package must contain
-an __init__.py, and a message bundle called 'pywikibot' containing
-messages. See L{twtranslate} for more information on the messages.
+L{set_message_package} with a package name. The package must contain an
+__init__.py, and a message bundle called 'pywikibot' containing messages.
+See L{twtranslate} for more information on the messages.
"""
#
# (C) Pywikibot team, 2004-2020
@@ -537,45 +536,50 @@
return self.source
def __iter__(self):
- raise NotImplementedError
+ raise NotImplementedError # pragma: no cover
def __len__(self):
- raise NotImplementedError
+ raise NotImplementedError # pragma: no cover
DEFAULT_FALLBACK = ('_default', )
def translate(code, xdict, parameters=None, fallback=False):
- """Return the most appropriate translation from a translation dict.
+ """Return the most appropriate localization from a localization dict.
- Given a language code and a dictionary, returns the dictionary's value for
+ Given a site code and a dictionary, returns the dictionary's value for
key 'code' if this key exists; otherwise tries to return a value for an
- alternative language that is most applicable to use on the wiki in
- language 'code' except fallback is False.
+ alternative code that is most applicable to use on the wiki in language
+ 'code' except fallback is False.
- The language itself is always checked first, then languages that
- have been defined to be alternatives, and finally English. If none of
- the options gives result, we just take the one language from xdict which
- may not be always the same. When fallback is iterable it'll return None if
- no code applies (instead of returning one).
+ The code itself is always checked first, then these codes that have
+ been defined to be alternatives, and finally English.
+
+ If fallback is False and the code is not found in the
For PLURAL support have a look at the twtranslate method.
- @param code: The language code
+ @param code: The site code as string or Site object. If xdict is an
+ extended dictionary the Site object should be used in favour of the
+ code string. Otherwise localizations from a wrong family might be
+ used.
@type code: str or Site object
- @param xdict: dictionary with language codes as keys or extended dictionary
- with family names as keys containing language dictionaries or
- a single (unicode) string. May contain PLURAL tags as
- described in twtranslate
+ @param xdict: dictionary with language codes as keys or extended
+ dictionary with family names as keys containing code dictionaries
+ or a single string. May contain PLURAL tags as described in
+ twtranslate
@type xdict: dict, string, unicode
@param parameters: For passing (plural) parameters
@type parameters: dict, string, unicode, int
@param fallback: Try an alternate language code. If it's iterable it'll
also try those entries and choose the first match.
@type fallback: boolean or iterable
- @raise IndexError: If the language supports and requires more plurals than
- defined for the given translation template.
+ @return: the localized string
+ @rtype: str
+ @raise IndexError: If the language supports and requires more plurals
+ than defined for the given PLURAL pattern.
+ @raise KeyError: No fallback key found if fallback is not False
"""
family = pywikibot.config.family
# If a site is given instead of a code, use its language
@@ -583,17 +587,19 @@
family = code.family.name
code = code.code
- # Check whether xdict has multiple projects
- if isinstance(xdict, dict):
- if family in xdict:
- xdict = xdict[family]
- elif 'wikipedia' in xdict:
- xdict = xdict['wikipedia']
+ try:
+ lookup = xdict[code]
+ except KeyError:
+ # Check whether xdict has multiple projects
+ if isinstance(xdict, dict) and family in xdict:
+ lookup = xdict[family]
+ else:
+ lookup = xdict
# Get the translated string
- if not isinstance(xdict, dict):
- trans = xdict
- elif not xdict:
+ if not isinstance(lookup, dict):
+ trans = lookup
+ elif not lookup:
trans = None
else:
codes = [code]
@@ -602,16 +608,22 @@
elif fallback is not False:
codes.extend(fallback)
for code in codes:
- if code in xdict:
- trans = xdict[code]
+ if code in lookup:
+ trans = lookup[code]
break
else:
- if fallback is False:
- return None
- raise KeyError('No fallback key found in lookup dict for "{}"'
- .format(code))
+ if fallback is not False:
+ raise KeyError('No fallback key found in lookup dict for "{}"'
+ .format(code))
+ trans = None
+
if trans is None:
+ if 'wikipedia' in xdict:
+ # fallback to wikipedia family
+ return translate(code, xdict['wikipedia'],
+ parameters=parameters, fallback=fallback)
return None # return None if we have no translation found
+
if parameters is None:
return trans
@@ -629,7 +641,7 @@
# On error: parameter is for PLURAL variants only,
# don't change the string
with suppress(KeyError, TypeError):
- return trans % parameters
+ trans = trans % parameters
return trans
diff --git a/tests/i18n_tests.py b/tests/i18n_tests.py
index 9b4c515..2cb6e7e 100644
--- a/tests/i18n_tests.py
+++ b/tests/i18n_tests.py
@@ -9,30 +9,166 @@
import pywikibot
-from pywikibot import i18n, bot, plural
+from pywikibot import bot, i18n, plural
from tests.aspects import (
- unittest, TestCase, DefaultSiteTestCase, PwbTestCase,
AutoDeprecationTestCase,
+ DefaultSiteTestCase,
+ PwbTestCase,
+ TestCase,
+ unittest,
)
+class Site:
+
+ """An object holding code and family, duck typing a pywikibot Site."""
+
+ class Family:
+
+ """Nested class to hold the family name attribute."""
+
+ pass
+
+ def __init__(self, code, family='wikipedia'):
+ """Initializer."""
+ self.code = code
+ self.family = self.Family()
+ setattr(self.family, 'name', family)
+
+ def __repr__(self):
+ return "'{site.family.name}:{site.code}'".format(site=self)
+
+
class TestTranslate(TestCase):
- """Test translate method."""
+ """Test translate method with fallback True."""
net = False
- def setUp(self):
- """Set up test method."""
- self.msg_localized = {'en': 'test-localized EN',
- 'nl': 'test-localized NL',
- 'fy': 'test-localized FY'}
- self.msg_semi_localized = {'en': 'test-semi-localized EN',
- 'nl': 'test-semi-localized NL'}
- self.msg_non_localized = {'en': 'test-non-localized EN'}
- self.msg_no_english = {'ja': 'test-no-english JA'}
- super().setUp()
+ xdict = {
+ 'en': 'test-localized EN',
+ 'commons': 'test-localized COMMONS',
+ 'wikipedia': {
+ 'nl': 'test-localized WP-NL',
+ 'fy': 'test-localized WP-FY',
+ 'wikipedia': { # test a deeply nested xdict
+ 'de': 'test-localized WP-DE',
+ },
+ },
+ 'wikisource': {
+ 'en': 'test-localized WS-EN',
+ 'fy': 'test-localized WS-FY',
+ 'ja': 'test-localized WS-JA',
+ },
+ }
+
+ def test_translate_commons(self):
+ """Test localization with xdict for commons.
+
+ Test whether the localzation is found either with the Site object
+ or with the site code.
+ """
+ site = Site('commons', 'commons')
+ for code in (site, 'commons'):
+ with self.subTest(code=code):
+ self.assertEqual(i18n.translate(code, self.xdict),
+ 'test-localized COMMONS')
+
+ def test_translate_de(self):
+ """Test localization fallbacks for 'de' with xdict.
+
+ 'de' key is defined in a nested 'wikipedia' sub dict. This should
+ always fall back to this nested 'wikipedia' entry.
+ """
+ site1 = Site('de', 'wikipedia')
+ site2 = Site('de', 'wikibooks')
+ site3 = Site('de', 'wikisource')
+ for code in (site1, site2, site3, 'de'):
+ with self.subTest(code=code):
+ self.assertEqual(i18n.translate(code, self.xdict),
+ 'test-localized WP-DE')
+
+ def test_translate_en(self):
+ """Test localization fallbacks for 'en' with xdict.
+
+ 'en' key is defined directly in xdict. This topmost key goes over
+ site specific key. Therefore 'test-localized WS-EN' is not given
+ back.
+ """
+ site1 = Site('en', 'wikipedia')
+ site2 = Site('en', 'wikibooks')
+ site3 = Site('en', 'wikisource')
+ for code in (site1, site2, site3, 'en'):
+ with self.subTest(code=code):
+ self.assertEqual(i18n.translate(code, self.xdict),
+ 'test-localized EN')
+
+ def test_translate_fy(self):
+ """Test localization fallbacks for 'fy' with xdict.
+
+ 'fy' key is defined in 'wikipedia' and 'wikisource' sub dicts.
+ They should have different localizations for these two families but
+ 'wikisource' should have a fallback to the 'wikipedia' entry.
+
+ Note: If the translate code is given as string, the result depends
+ on the current config.family entry. Therefore there is no test with
+ the code given as string.
+ """
+ site1 = Site('fy', 'wikipedia')
+ site2 = Site('fy', 'wikibooks')
+ site3 = Site('fy', 'wikisource')
+ for code in (site1, site2):
+ with self.subTest(code=code):
+ self.assertEqual(i18n.translate(code, self.xdict),
+ 'test-localized WP-FY')
+ self.assertEqual(i18n.translate(site3, self.xdict),
+ 'test-localized WS-FY')
+
+ def test_translate_nl(self):
+ """Test localization fallbacks for 'nl' with xdict.
+
+ 'nl' key is defined in 'wikipedia' sub dict. Therefore all
+ localizations have a fallback to the 'wikipedia' entry.
+ """
+ site1 = Site('nl', 'wikipedia')
+ site2 = Site('nl', 'wikibooks')
+ site3 = Site('nl', 'wikisource')
+ for code in (site1, site2, site3, 'nl'):
+ with self.subTest(code=code):
+ self.assertEqual(i18n.translate(code, self.xdict),
+ 'test-localized WP-NL')
+
+ def test_translate_ja(self):
+ """Test localization fallbacks for 'ja' with xdict.
+
+ 'ja' key is defined in 'wkisource' sub dict only. Therefore there
+ is no fallback to the 'wikipedia' entry and the localization result
+ is None.
+ """
+ site1 = Site('ja', 'wikipedia')
+ site2 = Site('ja', 'wikibooks')
+ site3 = Site('ja', 'wikisource')
+ for code in (site1, site2):
+ with self.subTest(code=code):
+ self.assertIsNone(i18n.translate(code, self.xdict))
+ self.assertEqual(i18n.translate(site3, self.xdict),
+ 'test-localized WS-JA')
+
+
+class TestFallbackTranslate(TestCase):
+
+ """Test translate method with fallback True."""
+
+ net = False
+
+ msg_localized = {'en': 'test-localized EN',
+ 'nl': 'test-localized NL',
+ 'fy': 'test-localized FY'}
+ msg_semi_localized = {'en': 'test-semi-localized EN',
+ 'nl': 'test-semi-localized NL'}
+ msg_non_localized = {'en': 'test-non-localized EN'}
+ msg_no_english = {'ja': 'test-no-english JA'}
def test_localized(self):
"""Test fully localized translations."""
To view, visit change 606671. To unsubscribe, or for help writing mail filters, visit settings.