explicitly map 'C' locale and other known definitions to LanguageTag

Starting up a dbgutil build with LC_ALL=C gave

i18nlangtag/source/languagetag/languagetag.cxx:1362:
LanguageTagImpl::convertLocaleToLang: with bAllowOnTheFlyID invalid 'C'
i18nlangtag/source/languagetag/languagetag.cxx:1533:
LanguageTag::getLanguageFromLangtag: pLangT==NULL for 'C'
i18nlangtag/source/languagetag/languagetag.cxx:1593:
LanguageTag::getRegionFromLangtag: pRegionT==NULL for 'C'

Nothing harmful in this case as the default fallback is 'en-US', but it
also indicated that other known non-standard "locales" such as
"sr-latin" or "german" were not resolved to the defined values. Likely
such weird values are not in use anymore, but..

Change-Id: Ib3469354ceb236552540da5fd11d8f9e9c5ab1fd

i18nlangtag/source/isolang/isolang.cxx

@@ -1259,27 +1259,31 @@ css::lang::Locale MsLangId::Conversion::getOverride( const css::lang::Locale& rL
 
 // static
 LanguageType MsLangId::Conversion::convertIsoNamesToLanguage( const OUString& rLang,
-        const OUString& rCountry )
+        const OUString& rCountry, bool bSkipIsoTable )
 {
     // language is lower case in table
     OUString aLowerLang = rLang.toAsciiLowerCase();
     // country is upper case in table
     OUString aUpperCountry = rCountry.toAsciiUpperCase();
 
-    //  first look for exact match
     const IsoLanguageCountryEntry* pFirstLang = nullptr;
-    for (const IsoLanguageCountryEntry* pEntry = aImplIsoLangEntries;
-            pEntry->mnLang != LANGUAGE_DONTKNOW; ++pEntry)
+
+    if (!bSkipIsoTable)
     {
-        if ( aLowerLang.equalsAscii( pEntry->maLanguage ) )
+        //  first look for exact match
+        for (const IsoLanguageCountryEntry* pEntry = aImplIsoLangEntries;
+                pEntry->mnLang != LANGUAGE_DONTKNOW; ++pEntry)
         {
-            if ( aUpperCountry.isEmpty() ||
-                 aUpperCountry.equalsAscii( pEntry->maCountry ) )
-                return pEntry->mnLang;
-            if ( !pFirstLang )
-                pFirstLang = pEntry;
-            else if ( !*pEntry->maCountry )
-                pFirstLang = pEntry;
+            if ( aLowerLang.equalsAscii( pEntry->maLanguage ) )
+            {
+                if ( aUpperCountry.isEmpty() ||
+                        aUpperCountry.equalsAscii( pEntry->maCountry ) )
+                    return pEntry->mnLang;
+                if ( !pFirstLang )
+                    pFirstLang = pEntry;
+                else if ( !*pEntry->maCountry )
+                    pFirstLang = pEntry;
+            }
         }
     }
 
@@ -1316,22 +1320,25 @@ LanguageType MsLangId::Conversion::convertIsoNamesToLanguage( const OUString& rL
         }
     }
 
-    // If the language is correct, then we return the default language
-    if ( pFirstLang )
-        return pFirstLang->mnLang;
-
-    //  if only the country is set, look for any entry matching the country
-    //  (to allow reading country and language in separate steps, in any order)
-    if ( !rCountry.isEmpty() && rLang.isEmpty() )
+    if (!bSkipIsoTable)
     {
-        for (const IsoLanguageCountryEntry* pEntry2 = aImplIsoLangEntries;
-                pEntry2->mnLang != LANGUAGE_DONTKNOW; ++pEntry2)
+        // If the language is correct, then we return the default language
+        if ( pFirstLang )
+            return pFirstLang->mnLang;
+
+        //  if only the country is set, look for any entry matching the country
+        //  (to allow reading country and language in separate steps, in any order)
+        if ( !rCountry.isEmpty() && rLang.isEmpty() )
         {
-            if ( aUpperCountry.equalsAscii( pEntry2->maCountry ) )
-                return pEntry2->mnLang;
-        }
+            for (const IsoLanguageCountryEntry* pEntry2 = aImplIsoLangEntries;
+                    pEntry2->mnLang != LANGUAGE_DONTKNOW; ++pEntry2)
+            {
+                if ( aUpperCountry.equalsAscii( pEntry2->maCountry ) )
+                    return pEntry2->mnLang;
+            }
 
-        aLowerLang = aUpperCountry.toAsciiLowerCase();
+            aLowerLang = aUpperCountry.toAsciiLowerCase();
+        }
     }
 
     // Look for privateuse definitions.
@@ -1357,7 +1364,7 @@ LanguageType MsLangId::Conversion::convertIsoNamesToLanguage( const OString& rLa
 {
     OUString aLang = OStringToOUString( rLang, RTL_TEXTENCODING_ASCII_US);
     OUString aCountry = OStringToOUString( rCountry, RTL_TEXTENCODING_ASCII_US);
-    return convertIsoNamesToLanguage( aLang, aCountry);
+    return convertIsoNamesToLanguage( aLang, aCountry, false);
 }
 
 

i18nlangtag/source/languagetag/languagetag.cxx

@@ -293,7 +293,7 @@ private:
     bool                isValidBcp47() const;
 
     void                convertLocaleToBcp47();
-    void                convertLocaleToLang( bool bAllowOnTheFlyID );
+    bool                convertLocaleToLang( bool bAllowOnTheFlyID );
     void                convertBcp47ToLocale();
     void                convertBcp47ToLang();
     void                convertLangToLocale();
@@ -1137,7 +1137,8 @@ bool LanguageTagImpl::canonicalize()
         {
             if (!mbInitializedLangID)
             {
-                convertLocaleToLang( false);
+                if (convertLocaleToLang( false))
+                    bChanged = true;
                 if (bTemporaryLocale || mnLangID == LANGUAGE_DONTKNOW)
                     bTemporaryLangID = true;
             }
@@ -1326,8 +1327,9 @@ void LanguageTagImpl::convertLocaleToBcp47()
 }
 
 
-void LanguageTagImpl::convertLocaleToLang( bool bAllowOnTheFlyID )
+bool LanguageTagImpl::convertLocaleToLang( bool bAllowOnTheFlyID )
 {
+    bool bRemapped = false;
     if (mbSystemLocale)
     {
         mnLangID = MsLangId::getRealLanguage( LANGUAGE_SYSTEM);
@@ -1335,6 +1337,24 @@ void LanguageTagImpl::convertLocaleToLang( bool bAllowOnTheFlyID )
     else
     {
         mnLangID = MsLangId::Conversion::convertLocaleToLanguage( maLocale);
+        if (mnLangID == LANGUAGE_DONTKNOW)
+        {
+            // convertLocaleToLanguage() only searches in ISO and private
+            // definitions, search in remaining definitions, i.e. for the "C"
+            // locale and non-standard things like "sr-latin" or "german" to
+            // resolve to a known locale, skipping ISO lll-CC that were already
+            // searched.
+            mnLangID = MsLangId::Conversion::convertIsoNamesToLanguage( maLocale.Language, maLocale.Country, true);
+            if (mnLangID != LANGUAGE_DONTKNOW)
+            {
+                // If one found, convert back and adapt Locale and Bcp47
+                // strings so we have a matching entry.
+                OUString aOrgBcp47( maBcp47);
+                convertLangToLocale();
+                convertLocaleToBcp47();
+                bRemapped = (maBcp47 != aOrgBcp47);
+            }
+        }
         if (mnLangID == LANGUAGE_DONTKNOW && bAllowOnTheFlyID)
         {
             if (isValidBcp47())
@@ -1364,6 +1384,7 @@ void LanguageTagImpl::convertLocaleToLang( bool bAllowOnTheFlyID )
         }
     }
     mbInitializedLangID = true;
+    return bRemapped;
 }
 
 

include/i18nlangtag/mslangid.hxx

@@ -249,9 +249,9 @@ public:
         I18NLANGTAG_DLLPRIVATE static css::lang::Locale getOverride(
                 const css::lang::Locale & rLocale );
 
-        /** Used by convertLocaleToLanguage(Locale) */
+        /** Used by convertLocaleToLanguageImpl(Locale) and LanguageTagImpl::convertLocaleToLang() */
         I18NLANGTAG_DLLPRIVATE static LanguageType convertIsoNamesToLanguage(
-                const OUString& rLang, const OUString& rCountry );
+                const OUString& rLang, const OUString& rCountry, bool bSkipIsoTable );
 
 
         /** Used by convertUnxByteStringToLanguage(OString) */