Merge branch 'dev' into python3

Conflicts:
	python/phonenumbers/geocoder.py
	python/tests/geocodertest.py

3 files changed, 61 insertions, 42 deletions

diff --git a/python/phonenumbers/__init__.py b/python/phonenumbers/__init__.py
index 3becc08e..0f44fdb3 100644
--- a/python/phonenumbers/__init__.py
+++ b/python/phonenumbers/__init__.py
@@ -120,24 +120,6 @@ from .phonenumbermatcher import PhoneNumberMatch, PhoneNumberMatcher, Leniency
 
 # The geodata occupies a lot of space, so only perform the import on first use
 # of geocoder functionality.
-def area_description_for_number(*args, **kwargs):
-    """Return a text description of the area of a PhoneNumber for the given language.
-
-    Arguments:
-    numobj -- The PhoneNumber object for which we want to get a text description.
-    lang -- A 2-letter lowercase ISO 639-1 language code for the language in
-                  which the description should be returned (e.g. "en")
-    script -- A 4-letter titlecase (first letter uppercase, rest lowercase)
-                  ISO script code as defined in ISO 15924, separated by an
-                  underscore (e.g. "Hant")
-    region --  A 2-letter uppercase ISO 3166-1 country code (e.g. "GB")
-
-    Returns a text description in the given language code, for the given phone
-    number's area, or an empty string if no description is available."""
-    from .geocoder import area_description_for_number as real_fn
-    return real_fn(*args, **kwargs)
-
-
 def country_name_for_number(*args, **kwargs):
     """Return the given PhoneNumber object's country name in the given language.
 
@@ -272,7 +254,6 @@ __all__ = ['PhoneNumber', 'CountryCodeSource', 'FrozenPhoneNumber',
            # end of items from phonenumberutil.py
            'connects_to_emergency_number', 'is_emergency_number', 'ShortNumberCost',
            'PhoneNumberMatch', 'PhoneNumberMatcher', 'Leniency',
-           'area_description_for_number',
            'country_name_for_number',
            'description_for_number',
            'description_for_valid_number',
diff --git a/python/phonenumbers/geocoder.py b/python/phonenumbers/geocoder.py
index f26c2489..6dc46e46 100644
--- a/python/phonenumbers/geocoder.py
+++ b/python/phonenumbers/geocoder.py
@@ -1,30 +1,30 @@
 """Phone number geocoding functionality
 
 >>> import phonenumbers
->>> from phonenumbers.geocoder import area_description_for_number
+>>> from phonenumbers.geocoder import description_for_number
 >>> from phonenumbers.util import u
 >>> gb_number = phonenumbers.parse("+442083612345", "GB")
 >>> de_number = phonenumbers.parse("0891234567", "DE")
 >>> ch_number = phonenumbers.parse("0431234567", "CH")
->>> str(area_description_for_number(gb_number, "en"))
+>>> str(description_for_number(gb_number, "en"))
 'London'
->>> str(area_description_for_number(gb_number, "fr"))  # fall back to English
+>>> str(description_for_number(gb_number, "fr"))  # fall back to English
 'London'
->>> str(area_description_for_number(gb_number, "en", region="GB"))
+>>> str(description_for_number(gb_number, "en", region="GB"))
 'London'
->>> str(area_description_for_number(gb_number, "en", region="US"))
-'London'
->>> str(area_description_for_number(de_number, "en"))
+>>> str(description_for_number(gb_number, "en", region="US"))  # fall back to country
+'United Kingdom'
+>>> str(description_for_number(de_number, "en"))
 'Munich'
->>> u('M\u00fcnchen') == area_description_for_number(de_number, "de")
+>>> u('M\u00fcnchen') == description_for_number(de_number, "de")
 True
->>> u('Z\u00fcrich') == area_description_for_number(ch_number, "de")
+>>> u('Z\u00fcrich') == description_for_number(ch_number, "de")
 True
->>> str(area_description_for_number(ch_number, "en"))
+>>> str(description_for_number(ch_number, "en"))
 'Zurich'
->>> str(area_description_for_number(ch_number, "fr"))
+>>> str(description_for_number(ch_number, "fr"))
 'Zurich'
->>> str(area_description_for_number(ch_number, "it"))
+>>> str(description_for_number(ch_number, "it"))
 'Zurigo'
 
 """
@@ -66,6 +66,9 @@ except ImportError:  # pragma no cover
         raise
 
 
+_LOCALE_NORMALIZATION_MAP = {"zh_TW": "zh_Hant", "zh_HK": "zh_Hant", "zh_MO": "zh_Hant"}
+
+
 def _may_fall_back_to_english(lang):
     # Don't fall back to English if the requested language is among the following:
     # - Chinese
@@ -74,16 +77,38 @@ def _may_fall_back_to_english(lang):
     return lang != "zh" and lang != "ja" and lang != "ko"
 
 
+def _full_locale(lang, script, region):
+    if script is not None:
+        if region is not None:
+            return "%s_%s_%s" % (lang, script, region)
+        else:
+            return "%s_%s" % (lang, script)
+    elif region is not None:
+        return "%s_%s" % (lang, region)
+    else:
+        return lang
+
+
 def _find_lang(langdict, lang, script, region):
     """Return the entry in the dictionary for the given language information."""
-    # First look for lang, script as a combination
-    lang_script = "%s_%s" % (lang, script)
-    if lang_script in langdict:
-        return langdict[lang_script]
+    # Check if we should map this to a different locale.
+    full_locale = _full_locale(lang, script, region)
+    if (full_locale in _LOCALE_NORMALIZATION_MAP and
+        _LOCALE_NORMALIZATION_MAP[full_locale] in langdict):
+        return langdict[_LOCALE_NORMALIZATION_MAP[full_locale]]
+    # First look for the full locale
+    if full_locale in langdict:
+        return langdict[full_locale]
+    # Then look for lang, script as a combination
+    if script is not None:
+        lang_script = "%s_%s" % (lang, script)
+        if lang_script in langdict:
+            return langdict[lang_script]
     # Next look for lang, region as a combination
-    lang_region = "%s_%s" % (lang, region)
-    if lang_region in langdict:
-        return langdict[lang_region]
+    if region is not None:
+        lang_region = "%s_%s" % (lang, region)
+        if lang_region in langdict:
+            return langdict[lang_region]
     # Fall back to bare language code lookup
     if lang in langdict:
         return langdict[lang]
@@ -94,7 +119,7 @@ def _find_lang(langdict, lang, script, region):
         return None
 
 
-def area_description_for_number(numobj, lang, script=None, region=None):
+def _area_description_for_number(numobj, lang, script=None, region=None):
     """Return a text description of the area of a PhoneNumber for the given language.
 
     Arguments:
@@ -199,7 +224,7 @@ def description_for_valid_number(numobj, lang, script=None, region=None):
     number, or an empty string if no description is available."""
     number_region = region_code_for_number(numobj)
     if region is None or region == number_region:
-        area_description = area_description_for_number(numobj, lang, script, region)
+        area_description = _area_description_for_number(numobj, lang, script, region)
         if area_description != "":
             return area_description
         else:
diff --git a/python/tests/geocodertest.py b/python/tests/geocodertest.py
index 3375fe5b..11b9b83e 100644
--- a/python/tests/geocodertest.py
+++ b/python/tests/geocodertest.py
@@ -22,7 +22,8 @@ import unittest
 from phonenumbers import PhoneNumber, FrozenPhoneNumber
 from phonenumbers import geocoder
 from phonenumbers import description_for_number, country_name_for_number
-from phonenumbers import description_for_valid_number, area_description_for_number
+from phonenumbers import description_for_valid_number
+from phonenumbers.geocoder import _area_description_for_number
 from phonenumbers.util import u
 
 # Allow override library geocoding metadata with the test metadata.
@@ -162,13 +163,14 @@ class PhoneNumberGeocoderTest(unittest.TestCase):
         invalid_number = PhoneNumber(country_code=210, national_number=123456)
         self.assertEqual("", country_name_for_number(invalid_number, "en"))
         # Ensure we exercise all public entrypoints directly
-        self.assertEqual("CA", area_description_for_number(US_NUMBER1, "en"))
+        self.assertEqual("CA", _area_description_for_number(US_NUMBER1, "en"))
         self.assertEqual("CA", description_for_valid_number(US_NUMBER1, "en"))
         self.assertEqual("", description_for_valid_number(US_INVALID_NUMBER, "en"))
         # Add in some script and region specific fictional names
         TEST_GEOCODE_DATA['1650960'] = {'en': u("Mountain View, CA"),
                                         "en_GB": u("Mountain View California"),
                                         "en_US": u("Mountain View, Sunny California"),
+                                        "en_Xyzz_US": u("MTV - xyzz"),
                                         "en_Latn": u("MountainView")}
         # The following test might one day return "Mountain View California"
         self.assertEqual("United States",
@@ -179,7 +181,18 @@ class PhoneNumberGeocoderTest(unittest.TestCase):
                          description_for_number(US_NUMBER2, _ENGLISH, script="Latn"))
         self.assertEqual("United States",
                          description_for_number(US_NUMBER2, _ENGLISH, script="Latn", region="GB"))
+        self.assertEqual("MTV - xyzz",
+                         description_for_number(US_NUMBER2, _ENGLISH, script="Xyzz", region="US"))
+        self.assertEqual("Mountain View, Sunny California",
+                         description_for_number(US_NUMBER2, _ENGLISH, script="Zazz", region="US"))
         # Get a different result when there is a script-specific variant
         self.assertEqual("MountainView",
                          description_for_number(US_NUMBER2, _ENGLISH, script="Latn", region="US"))
         TEST_GEOCODE_DATA['1650960'] = {'en': u("Mountain View, CA")}
+
+        # Test the locale mapping
+        TEST_GEOCODE_DATA['8868'] = {'zh': u("Chinese"), 'zh_Hant': u("Hant-specific")}
+        tw_number = FrozenPhoneNumber(country_code=886, national_number=810080123)
+        self.assertEqual("Hant-specific",
+                         description_for_number(tw_number, "zh", region="TW"))
+        del TEST_GEOCODE_DATA['8868']