bryce/mediawiki-extensions-UniversalLanguageSelector
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding language presence by territory to langdb

Browse Source 
A very simple mechanism for importing per-country language lists
from CLDR to ULS' langdb.

If I understand correctly, we only need languages spoken in a country
ordered by number of speakers. The CLDR data already has it and it should be
mostly useful.

Also added a utility function and a test.

Some tweaks to override the CLDR data are still needed:

* The data as it is omits some useful languages. For example, Amharic is not
  listed in Eritrea.
* Some countries have a very large number of languages. Ideally it's right,
  but is not practical currently, for example India with 75. Maybe
  hand-picking or limiting the choice to top X languages can be useful,
  but requires thought.
* Some language codes are standard, but different from Wikipedia practice,
  for example "pa_Guru" (we just write "pa"). Maybe a mapping of codes
  is needed.

Change-Id: I3c0cd5a9118997ba39a4f3695978e359f3de6956
This commit is contained in:
Amir E. Aharoni committed 2012-08-26 10:40:52 +03:00
1 parent 0d73ffe09e
commit ca411138c7
4 files changed
+62 -5
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/jquery.uls/data/ulsdata2json.php
+49 -3
View File
@@ -20,9 +20,53 @@
include __DIR__ . '/spyc.php';
$data = file_get_contents( 'langdb.yaml' );
$parsed = spyc_load( $data );
$json = json_encode( $parsed );
print( "Reading langdb.yaml...\n" );
$yamlLangdb = file_get_contents( 'langdb.yaml' );
$parsedLangdb = spyc_load( $yamlLangdb );
$supplementalDataFilename = 'supplementalData.xml';
$supplementalDataUrl = "http://unicode.org/repos/cldr/trunk/common/supplemental/$supplementalDataFilename";
$curl = curl_init( $supplementalDataUrl );
$supplementalDataFile = fopen( $supplementalDataFilename, 'w' );
curl_setopt( $curl, CURLOPT_FILE, $supplementalDataFile );
curl_setopt( $curl, CURLOPT_HEADER, 0 );
print( "Trying to download $supplementalDataUrl...\n" );
$curlSuccess = curl_exec( $curl );
curl_close( $curl );
fclose( $supplementalDataFile );
if ( !$curlSuccess ) {
die( "Failed to download CLDR data from $supplementalDataUrl.\n" );
}
print( "Downloaded $supplementalDataFilename, trying to parse...\n" );
$supplementalData = simplexml_load_file( $supplementalDataFilename );
if ( !( $supplementalData instanceof SimpleXMLElement ) ) {
die( "Attempt to load CLDR data from $supplementalDataFilename failed.\n" );
}
print( "CLDR supplemental data parsed successfully, reading territories info...\n" );
$parsedLangdb['territories'] = array();
foreach ( $supplementalData->territoryInfo->territory as $territoryRecord ) {
$territoryAtributes = $territoryRecord->attributes();
$territoryCodeAttr = $territoryAtributes['type'];
$territoryCode = "$territoryCodeAttr[0]";
$parsedLangdb['territories'][$territoryCode] = array();
foreach ( $territoryRecord->languagePopulation as $languageRecord ) {
$languageAttributes = $languageRecord->attributes();
$languageCodeAttr = $languageAttributes['type'];
$parsedLangdb['territories'][$territoryCode][] = "$languageCodeAttr[0]";
}
}
print( "Writing JSON langdb...\n" );
$json = json_encode( $parsedLangdb );
$js = <<<JAVASCRIPT
// Please do not edit. This file is generated from data/langdb.yaml by ulsdata2json.php
( function ( $ ) {
@@ -32,3 +76,5 @@ $js = <<<JAVASCRIPT
JAVASCRIPT;
file_put_contents( '../src/jquery.uls.data.js', $js );
print( "Done.\n" );
lib/jquery.uls/src/jquery.uls.data.js
+1 -1
View File
@@ -1,5 +1,5 @@
// Please do not edit. This file is generated from data/langdb.yaml by ulsdata2json.php
( function ( $ ) {
$.uls = $.uls || {};
$.uls.data = {"languages":{"aa":["Latn",["AF"],"Qaf\u00e1r af"],"ab":["Cyrl",["EU"],"\u0410\u04a7\u0441\u0448\u04d9\u0430"],"ace":["Latn",["AS","PA"],"Ac\u00e8h"],"ady-cyrl":["Cyrl",["EU"],"\u0410\u0434\u044b\u0433\u044d\u0431\u0437\u044d"],"ady-latn":["Latn",["EU"],"Adygabze"],"ady":["Cyrl",["EU"],"\u0410\u0434\u044b\u0433\u044d\u0431\u0437\u044d"],"aeb":["Arab",["AF"],"\u0632\u064e\u0648\u064f\u0646"],"af":["Latn",["AF"],"Afrikaans"],"ahr":["Deva",["AS"],"\u0905\u0939\u093f\u0930\u093e\u0923\u0940"],"ak":["Latn",["AF"],"Akan"],"akz":["Latn",["NA"],"Albaamo innaa\u026ciilka"],"aln":["Latn",["EU"],"Geg\u00eb"],"am":["Ethi",["AF"],"\u12a0\u121b\u122d\u129b"],"an":["Latn",["EU"],"aragon\u00e9s"],"ang":["Latn",["EU"],"\u00c6nglisc"],"anp":["Deva",["AS"],"\u0905\u0919\u094d\u0917\u093f\u0915\u093e"],"ar":["Arab",["ME"],"\u0627\u0644\u0639\u0631\u0628\u064a\u0629"],"arc":["Syrc",["ME"],"\u0710\u072a\u0721\u071d\u0710"],"arn":["Latn",["SA"],"mapudungun"],"aro":["Latn",["LA"],"Araona"],"arq":["Latn",["AF"],"Dziri"],"ary":["Latn",["ME"],"Ma\u0121ribi"],"arz":["Arab",["ME"],"\u0645\u0635\u0631\u0649"],"as":["Beng",["AS"],"\u0985\u09b8\u09ae\u09c0\u09df\u09be"],"ase":["Sgnw",["NA"],"American sign language"],"ast":["Latn",["EU"],"asturianu"],"av":["Cyrl",["EU"],"\u0430\u0432\u0430\u0440"],"avk":["Latn",["WW"],"Kotava"],"ay":["Latn",["SA"],"Aymar aru"],"az":["Latn",["EU","ME"],"az\u0259rbaycanca"],"ba":["Cyrl",["EU"],"\u0431\u0430\u0448\u04a1\u043e\u0440\u0442\u0441\u0430"],"bar":["Latn",["EU"],"Boarisch"],"bbc-latn":["Latn",["AS"],"Batak Toba"],"bbc":["Batk",["AS"],"Batak Toba\/Batak autonym unknown"],"bcc":["Arab",["AS","ME"],"\u0628\u0644\u0648\u0686\u06cc \u0645\u06a9\u0631\u0627\u0646\u06cc"],"bcl":["Latn",["AS"],"Bikol Central"],"be-tarask":["Cyrl",["EU"],"\u0431\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f (\u0442\u0430\u0440\u0430\u0448\u043a\u0435\u0432\u0456\u0446\u0430)"],"be-x-old":["Cyrl",["EU"],"\u0431\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f (\u0442\u0430\u0440\u0430\u0448\u043a\u0435\u0432\u0456\u0446\u0430)"],"be":["Cyrl",["EU"],"\u0431\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f"],"bew":["Latn",["AS"],"Bahasa Betawi"],"bfq":["Taml",["AS"],"\u0baa\u0b9f\u0b95\u0bbe"],"bg":["Cyrl",["EU"],"\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438"],"bh":["Deva",["AS"],"\u092d\u094b\u091c\u092a\u0941\u0930\u0940"],"bho":["Deva",["AS"],"\u092d\u094b\u091c\u092a\u0941\u0930\u0940"],"bi":["Latn",["PA"],"Bislama"],"bjn":["Latn",["AS"],"Bahasa Banjar"],"bm":["Latn",["AF"],"bamanankan"],"bn":["Beng",["AS"],"\u09ac\u09be\u0982\u09b2\u09be"],"bo":["Tibt",["AS"],"\u0f56\u0f7c\u0f51\u0f0b\u0f61\u0f72\u0f42"],"bpy":["Beng",["AS"],"\u09ac\u09bf\u09b7\u09cd\u09a3\u09c1\u09aa\u09cd\u09b0\u09bf\u09af\u09bc\u09be \u09ae\u09a3\u09bf\u09aa\u09c1\u09b0\u09c0"],"bqi":["Arab",["ME"],"\u0628\u062e\u062a\u064a\u0627\u0631\u064a"],"br":["Latn",["EU"],"brezhoneg"],"brh":["Latn",["ME","AS"],"Br\u00e1hu\u00ed"],"bs":["Latn",["EU"],"bosanski"],"bto":["Latn",["AS"],"Iriga Bicolano"],"bug":["Bugi",["AS"],"\u1a05\u1a14 \u1a15\u1a18\u1a01\u1a17"],"bxr":["Cyrl",["AS"],"\u0431\u0443\u0440\u044f\u0430\u0434"],"ca":["Latn",["EU"],"catal\u00e0"],"cbk-zam":["Latn",["AS"],"Chavacano de Zamboanga"],"cdo":["Latn",["AS"],"M\u00ecng-d\u0115\u0324ng-ng\u1e73\u0304"],"ce":["Cyrl",["EU"],"\u043d\u043e\u0445\u0447\u0438\u0439\u043d"],"ceb":["Latn",["AS"],"Cebuano"],"ch":["Latn",["PA"],"Chamoru"],"cho":["Latn",["NA"],"Choctaw"],"chr":["Cher",["NA"],"\u13e3\u13b3\u13a9"],"chy":["Latn",["NA"],"Tsets\u00eahest\u00e2hese"],"ckb":["Arab",["ME"],"\u06a9\u0648\u0631\u062f\u06cc"],"co":["Latn",["EU"],"corsu"],"cps":["Latn",["AS"],"Capice\u00f1o"],"cr-cans":["Cans",["NA"],"\u14c0\u1426\u1403\u152d\u140d\u140f\u1423"],"cr-latn":["Latn",["NA"],"N\u0113hiyaw\u0113win"],"cr":["Cans",["NA"],"N\u0113hiyaw\u0113win \/ \u14c0\u1426\u1403\u152d\u140d\u140f\u1423"],"crh-cyrl":["Cyrl",["EU"],"\u043a\u044a\u044b\u0440\u044b\u043c\u0442\u0430\u0442\u0430\u0440\u0434\u0436\u0430 (\u041a\u0438\u0440\u0438\u043b\u043b)"],"crh-latn":["Latn",["EU"],"q\u0131r\u0131mtatarca (Latin)"],"crh":["Latn",["EU"],"\u043a\u044a\u044b\u0440\u044b\u043c\u0442\u0430\u0442\u0430\u0440\u0434\u0436\u0430 \/ q\u0131r\u0131mtatarca"],"cs":["Latn",["EU"],"\u010desky"],"csb":["Latn",["EU"],"kasz\u00ebbsczi"],"cu":["Cyrl",["EU"],"\u0441\u043b\u043e\u0432\u0463\u0301\u043d\u044c\u0441\u043a\u044a \/ \u2c14\u2c0e\u2c11\u2c02\u2c21\u2c10\u2c20\u2c14\u2c0d\u2c1f"],"cv":["Cyrl",["EU"],"\u0427\u04d1\u0432\u0430\u0448\u043b\u0430"],"cy":["Latn",["EU"],"Cymraeg"],"da":["Latn",["EU"],"dansk"],"de-at":["Latn",["EU"],"\u00d6sterreichisches Deutsch"],"de-ch":["Latn",["EU"],"Schweizer Hochdeutsch"],"de-formal":["Latn",["EU"],"Deutsch (Sie-Form)"],"de":["Latn",["EU"],"Deutsch"],"diq":["Latn",["EU","AS"],"Zazaki"],"dsb":["Latn",["EU"],"dolnoserbski"],"dtp":["Latn",["AS"],"Dusun Bundu-liwan"],"dv":["Thaa",["AS"],"\u078b\u07a8\u0788\u07ac\u0780\u07a8\u0784\u07a6\u0790\u07b0"],"dz":["TibtLine truncated
$.uls.data = {"languages":{"aa":["Latn",["AF"],"Qaf\u00e1r af"],"ab":["Cyrl",["EU"],"\u0410\u04a7\u0441\u0448\u04d9\u0430"],"ace":["Latn",["AS","PA"],"Ac\u00e8h"],"ady-cyrl":["Cyrl",["EU"],"\u0410\u0434\u044b\u0433\u044d\u0431\u0437\u044d"],"ady-latn":["Latn",["EU"],"Adygabze"],"ady":["Cyrl",["EU"],"\u0410\u0434\u044b\u0433\u044d\u0431\u0437\u044d"],"aeb":["Arab",["AF"],"\u0632\u064e\u0648\u064f\u0646"],"af":["Latn",["AF"],"Afrikaans"],"ahr":["Deva",["AS"],"\u0905\u0939\u093f\u0930\u093e\u0923\u0940"],"ak":["Latn",["AF"],"Akan"],"akz":["Latn",["NA"],"Albaamo innaa\u026ciilka"],"aln":["Latn",["EU"],"Geg\u00eb"],"am":["Ethi",["AF"],"\u12a0\u121b\u122d\u129b"],"an":["Latn",["EU"],"aragon\u00e9s"],"ang":["Latn",["EU"],"\u00c6nglisc"],"anp":["Deva",["AS"],"\u0905\u0919\u094d\u0917\u093f\u0915\u093e"],"ar":["Arab",["ME"],"\u0627\u0644\u0639\u0631\u0628\u064a\u0629"],"arc":["Syrc",["ME"],"\u0710\u072a\u0721\u071d\u0710"],"arn":["Latn",["SA"],"mapudungun"],"aro":["Latn",["LA"],"Araona"],"arq":["Latn",["AF"],"Dziri"],"ary":["Latn",["ME"],"Ma\u0121ribi"],"arz":["Arab",["ME"],"\u0645\u0635\u0631\u0649"],"as":["Beng",["AS"],"\u0985\u09b8\u09ae\u09c0\u09df\u09be"],"ase":["Sgnw",["NA"],"American sign language"],"ast":["Latn",["EU"],"asturianu"],"av":["Cyrl",["EU"],"\u0430\u0432\u0430\u0440"],"avk":["Latn",["WW"],"Kotava"],"ay":["Latn",["SA"],"Aymar aru"],"az":["Latn",["EU","ME"],"az\u0259rbaycanca"],"ba":["Cyrl",["EU"],"\u0431\u0430\u0448\u04a1\u043e\u0440\u0442\u0441\u0430"],"bar":["Latn",["EU"],"Boarisch"],"bbc-latn":["Latn",["AS"],"Batak Toba"],"bbc":["Batk",["AS"],"Batak Toba\/Batak autonym unknown"],"bcc":["Arab",["AS","ME"],"\u0628\u0644\u0648\u0686\u06cc \u0645\u06a9\u0631\u0627\u0646\u06cc"],"bcl":["Latn",["AS"],"Bikol Central"],"be-tarask":["Cyrl",["EU"],"\u0431\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f (\u0442\u0430\u0440\u0430\u0448\u043a\u0435\u0432\u0456\u0446\u0430)"],"be-x-old":["Cyrl",["EU"],"\u0431\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f (\u0442\u0430\u0440\u0430\u0448\u043a\u0435\u0432\u0456\u0446\u0430)"],"be":["Cyrl",["EU"],"\u0431\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f"],"bew":["Latn",["AS"],"Bahasa Betawi"],"bfq":["Taml",["AS"],"\u0baa\u0b9f\u0b95\u0bbe"],"bg":["Cyrl",["EU"],"\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438"],"bh":["Deva",["AS"],"\u092d\u094b\u091c\u092a\u0941\u0930\u0940"],"bho":["Deva",["AS"],"\u092d\u094b\u091c\u092a\u0941\u0930\u0940"],"bi":["Latn",["PA"],"Bislama"],"bjn":["Latn",["AS"],"Bahasa Banjar"],"bm":["Latn",["AF"],"bamanankan"],"bn":["Beng",["AS"],"\u09ac\u09be\u0982\u09b2\u09be"],"bo":["Tibt",["AS"],"\u0f56\u0f7c\u0f51\u0f0b\u0f61\u0f72\u0f42"],"bpy":["Beng",["AS"],"\u09ac\u09bf\u09b7\u09cd\u09a3\u09c1\u09aa\u09cd\u09b0\u09bf\u09af\u09bc\u09be \u09ae\u09a3\u09bf\u09aa\u09c1\u09b0\u09c0"],"bqi":["Arab",["ME"],"\u0628\u062e\u062a\u064a\u0627\u0631\u064a"],"br":["Latn",["EU"],"brezhoneg"],"brh":["Latn",["ME","AS"],"Br\u00e1hu\u00ed"],"bs":["Latn",["EU"],"bosanski"],"bto":["Latn",["AS"],"Iriga Bicolano"],"bug":["Bugi",["AS"],"\u1a05\u1a14 \u1a15\u1a18\u1a01\u1a17"],"bxr":["Cyrl",["AS"],"\u0431\u0443\u0440\u044f\u0430\u0434"],"ca":["Latn",["EU"],"catal\u00e0"],"cbk-zam":["Latn",["AS"],"Chavacano de Zamboanga"],"cdo":["Latn",["AS"],"M\u00ecng-d\u0115\u0324ng-ng\u1e73\u0304"],"ce":["Cyrl",["EU"],"\u043d\u043e\u0445\u0447\u0438\u0439\u043d"],"ceb":["Latn",["AS"],"Cebuano"],"ch":["Latn",["PA"],"Chamoru"],"cho":["Latn",["NA"],"Choctaw"],"chr":["Cher",["NA"],"\u13e3\u13b3\u13a9"],"chy":["Latn",["NA"],"Tsets\u00eahest\u00e2hese"],"ckb":["Arab",["ME"],"\u06a9\u0648\u0631\u062f\u06cc"],"co":["Latn",["EU"],"corsu"],"cps":["Latn",["AS"],"Capice\u00f1o"],"cr-cans":["Cans",["NA"],"\u14c0\u1426\u1403\u152d\u140d\u140f\u1423"],"cr-latn":["Latn",["NA"],"N\u0113hiyaw\u0113win"],"cr":["Cans",["NA"],"N\u0113hiyaw\u0113win \/ \u14c0\u1426\u1403\u152d\u140d\u140f\u1423"],"crh-cyrl":["Cyrl",["EU"],"\u043a\u044a\u044b\u0440\u044b\u043c\u0442\u0430\u0442\u0430\u0440\u0434\u0436\u0430 (\u041a\u0438\u0440\u0438\u043b\u043b)"],"crh-latn":["Latn",["EU"],"q\u0131r\u0131mtatarca (Latin)"],"crh":["Latn",["EU"],"\u043a\u044a\u044b\u0440\u044b\u043c\u0442\u0430\u0442\u0430\u0440\u0434\u0436\u0430 \/ q\u0131r\u0131mtatarca"],"cs":["Latn",["EU"],"\u010desky"],"csb":["Latn",["EU"],"kasz\u00ebbsczi"],"cu":["Cyrl",["EU"],"\u0441\u043b\u043e\u0432\u0463\u0301\u043d\u044c\u0441\u043a\u044a \/ \u2c14\u2c0e\u2c11\u2c02\u2c21\u2c10\u2c20\u2c14\u2c0d\u2c1f"],"cv":["Cyrl",["EU"],"\u0427\u04d1\u0432\u0430\u0448\u043b\u0430"],"cy":["Latn",["EU"],"Cymraeg"],"da":["Latn",["EU"],"dansk"],"de-at":["Latn",["EU"],"\u00d6sterreichisches Deutsch"],"de-ch":["Latn",["EU"],"Schweizer Hochdeutsch"],"de-formal":["Latn",["EU"],"Deutsch (Sie-Form)"],"de":["Latn",["EU"],"Deutsch"],"diq":["Latn",["EU","AS"],"Zazaki"],"dsb":["Latn",["EU"],"dolnoserbski"],"dtp":["Latn",["AS"],"Dusun Bundu-liwan"],"dv":["Thaa",["AS"],"\u078b\u07a8\u0788\u07ac\u0780\u07a8\u0784\u07a6\u0790\u07b0"],"dz":["TibtLine truncated
} )( jQuery );
lib/jquery.uls/src/jquery.uls.data.utils.js
+9
View File
@@ -338,4 +338,13 @@
$.uls.data.isRtl = function( language ) {
return $.inArray( $.uls.data.script( language ), $.uls.data.rtlscripts ) !== -1;
};
/**
* Returns the languages spoken in a territory.
* @param string Territory code
* @return list of language codes
*/
$.uls.data.languagesInTerritory = function( territory ) {
return $.uls.data.territories[territory];
};
} )( jQuery );
tests/qunit/ext.uls.tests.js
+3 -1
View File
@@ -61,7 +61,7 @@ test( "-- Initial check", function() {
} );
test( "-- $.uls.data testing", function() {
expect( 23 );
expect( 24 );
strictEqual( $.uls.data.autonyms()['he'], 'עברית', 'Correct autonym is returned for Hebrew using autonyms().' );
@@ -128,6 +128,8 @@ test( "-- $.uls.data testing", function() {
strictEqual( $.uls.data.isRtl( "te" ), false, "Telugu language is not RTL" );
strictEqual( $.uls.data.isRtl( "dv" ), true, "Divehi language is RTL" );
ok( $.inArray( "sah", $.uls.data.languagesInTerritory( "RU" ) ) > -1, "Sakha language is spoken in Russia" );
} );
}() );