bryce/language-data
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #80 from Abijeet/feature/T218639-2

Browse Source 
Replace Travis with Github actions
This commit is contained in:
James Forrester
2020-01-31 16:05:58 -08:00
committed by GitHub
parent 6ca93c0966 7c970cb247
commit 6ca9f5288b
20 changed files with 3009 additions and 595 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.editorconfig Normal file
View File

@@ -0,0 +1,11 @@
root = true
[*]
indent_style = tab
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.md]
trim_trailing_whitespace = false

27
.github/workflows/node.yml vendored Normal file
View File

@@ -0,0 +1,27 @@
name: Node.js build
on:
[pull_request, push]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
node-v: [8.x, 10.x, 12.x]
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Use Node.js ${{ matrix.node-v }}
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-v }}
- name: Install package.json dependencies
run: npm install
- name: Run Node.js test suite
run: npm test
- name: Run linting
run: npm run lint

22
.github/workflows/php.yml vendored Normal file
View File

@@ -0,0 +1,22 @@
name: PHP build
on:
[pull_request, push]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
php: [7.2, 7.3]
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Install dependencies
run: composer install --prefer-dist --no-interaction --ansi
- name: Run tests
run: php${{ matrix.php }} ./vendor/bin/phpunit tests/php/*
- name: Running phpcs
run: composer run-script cs

2
.gitignore vendored
View File

@@ -14,3 +14,5 @@ yarn-error.log*
*.njsproj
*.sln
# PHP
vendor/

10
.phpcs.xml Normal file
View File

@@ -0,0 +1,10 @@
<?xml version="1.0"?>
<ruleset>
<rule ref="./vendor/mediawiki/mediawiki-codesniffer/MediaWiki">
<exclude-pattern type="relative">src/util/spyc.php</exclude-pattern>
</rule>
<file>src</file>
<file>tests/php</file>
<arg name="extensions" value="php,php5,inc"/>
<arg name="encoding" value="UTF-8"/>
</ruleset>

5
.travis.yml
View File

@@ -1,5 +0,0 @@
language: node_js
node_js:
- 8
sudo: false

179
GPL-LICENSE → COPYING
View File

@@ -1,65 +1,65 @@
== GNU GENERAL PUBLIC LICENSE ==
GNU GENERAL PUBLIC LICENSE
Version 2, June 1991
Version 2, June 1991
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Copyright (C) 1989, 1991 Free Software Foundation, Inc.
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
=== Preamble ===
The licenses for most software are designed to take away your
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
License is intended to guarantee your freedom to share and change free
software--to make sure the software is free for all its users. This
General Public License applies to most of the Free Software
Foundation's software and to any other program whose authors commit to
using it. (Some other Free Software Foundation software is covered by
the GNU Library General Public License instead.) You can apply it to
the GNU Lesser General Public License instead.) You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
this service if you wish), that you receive source code or can get it
if you want it, that you can change the software or use pieces of it
in new free programs; and that you know you can do these things.
To protect your rights, we need to make restrictions that forbid
To protect your rights, we need to make restrictions that forbid
anyone to deny you these rights or to ask you to surrender the rights.
These restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must give the recipients all the rights that
you have. You must make sure that they, too, receive or can get the
source code. And you must show them these terms so they know their
rights.
We protect your rights with two steps: (1) copyright the software, and
We protect your rights with two steps: (1) copyright the software, and
(2) offer you this license which gives you legal permission to copy,
distribute and/or modify the software.
Also, for each author's protection and ours, we want to make certain
Also, for each author's protection and ours, we want to make certain
that everyone understands that there is no warranty for this free
software. If the software is modified by someone else and passed on, we
want its recipients to know that what they have is not the original, so
that any problems introduced by others will not reflect on the original
authors' reputations.
Finally, any free program is threatened constantly by software
Finally, any free program is threatened constantly by software
patents. We wish to avoid the danger that redistributors of a free
program will individually obtain patent licenses, in effect making the
program proprietary. To prevent this, we have made it clear that any
patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and
The precise terms and conditions for copying, distribution and
modification follow.
== TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION ==
GNU GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
'''0.''' This License applies to any program or other work which contains
0. This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License. The "Program", below,
refers to any such program or work, and a "work based on the Program"
@@ -76,7 +76,7 @@ is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
'''1.''' You may copy and distribute verbatim copies of the Program's
1. You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
@@ -87,29 +87,29 @@ along with the Program.
You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.
'''2.''' You may modify your copy or copies of the Program or any portion
2. You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
'''a)''' You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
a) You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
'''b)''' You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
b) You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
'''c)''' If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
c) If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
@@ -131,26 +131,26 @@ with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
'''3.''' You may copy and distribute the Program (or a work based on it,
3. You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:
'''a)''' Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
a) Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
'''b)''' Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
b) Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
'''c)''' Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
c) Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for
making modifications to it. For an executable work, complete source
@@ -169,7 +169,7 @@ access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
'''4.''' You may not copy, modify, sublicense, or distribute the Program
4. You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
@@ -177,7 +177,7 @@ However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.
'''5.''' You are not required to accept this License, since you have not
5. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Program or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
@@ -186,7 +186,7 @@ Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.
'''6.''' Each time you redistribute the Program (or any work based on the
6. Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions. You may not impose any further
@@ -194,7 +194,7 @@ restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.
'''7.''' If, as a consequence of a court judgment or allegation of patent
7. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
@@ -226,7 +226,7 @@ impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
'''8.''' If the distribution and/or use of the Program is restricted in
8. If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
@@ -234,7 +234,7 @@ those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
'''9.''' The Free Software Foundation may publish revised and/or new versions
9. The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
@@ -247,7 +247,7 @@ Software Foundation. If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.
'''10.''' If you wish to incorporate parts of the Program into other free
10. If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission. For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
@@ -255,9 +255,9 @@ make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
=== NO WARRANTY ===
NO WARRANTY
'''11.''' BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
@@ -267,7 +267,7 @@ TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.
'''12.''' IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
@@ -277,47 +277,45 @@ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
'''END OF TERMS AND CONDITIONS'''
END OF TERMS AND CONDITIONS
== How to Apply These Terms to Your New Programs ==
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
Copyright (C) <year> <name of author>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Also add information on how to contact you by electronic and paper mail.
If the program is interactive, make it output a short notice like this
when it starts in an interactive mode:
Gnomovision version 69, Copyright (C) year name of author
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
Gnomovision version 69, Copyright (C) year name of author
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, the commands you use may
@@ -328,15 +326,14 @@ You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the program, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
`Gnomovision' (which makes passes at compilers) written by James Hacker.
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
`Gnomovision' (which makes passes at compilers) written by James Hacker.
<signature of Ty Coon>, 1 April 1989
Ty Coon, President of Vice
<signature of Ty Coon>, 1 April 1989
Ty Coon, President of Vice
This General Public License does not permit incorporating your program into
proprietary programs. If your program is a subroutine library, you may
consider it more useful to permit linking proprietary applications with the
library. If this is what you want to do, use the GNU Library General
library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License.

22
MIT-LICENSE
View File

@@ -1,22 +0,0 @@
Copyright (c) 2012 Alolita Sharma, Amir Aharoni, Arun Ganesh, Brandon Harris,
Niklas Laxström, Pau Giner, Santhosh Thottingal, Siebrand Mazeland and other
contributors. See CREDITS for a list.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

6
README.md
View File

@@ -13,7 +13,7 @@ The language data with following details are populated from the current version
## Adding languages
New languages must be added to the data/langdb.yaml file.
New languages must be added to the `data/langdb.yaml` file.
The file format is:
@@ -21,11 +21,11 @@ ISO 639 code: [writing system code, [regions list], autonym]
The writing system is indicated using ISO 15924 codes. Make sure that the code appears in the scriptgroups section towards the end of the file, and add it if it doesn't.
The list of region codes appears at the end of data/langdb.yaml.
The list of region codes appears at the end of `data/langdb.yaml`.
The autonym is the name of the language in the language itself. In some cases, for example for extinct languages such as Jewish Babylonian Aramaic (tmr), the name can be something that is useful for modern users, but in most cases it should be the natural name in the language itself. Please do your best to verify that it's spelled correctly in reliable sources.
After adding a language to data/langdb.yaml, run `php ulsdata2json.php` in the data/ directory to generate the language-data.json file. Don't edit language-data.json manually.
After adding a language to `data/langdb.yaml`, run `php src/util/ulsdata2json.php` in the base directory to generate the language-data.json file. Don't edit language-data.json manually.
Example:
`myv: [Cyrl, [EU], эрзянь]`

51
composer.json Normal file
View File

@@ -0,0 +1,51 @@
{
"name": "wikimedia/language-data",
"description": "CLDR based language data and utilities.",
"keywords": [
"cldr",
"cldr-data",
"internationalization",
"localization",
"l10n"
],
"license": "GPL-2.0-or-later",
"authors": [
{
"name": "Santhosh Thottingal",
"email": "santhosh.thottingal@gmail.com"
},
{
"name": "Amir Aharoni"
},
{
"name": "Niklas Laxström",
"email": "niklas.laxstrom@gmail.com"
}
],
"type": "library",
"require": {
"php": ">=7.2"
},
"require-dev": {
"phpunit/phpunit": "8.*",
"mediawiki/mediawiki-codesniffer": "29.0.0"
},
"support": {
"issues": "https://github.com/wikimedia/language-data/issues",
"irc": "irc://irc.freenode.net/mediawiki-i18n"
},
"autoload": {
"psr-4": {
"Wikimedia\\LanguageData\\": "src/"
}
},
"scripts": {
"fix": "phpcbf",
"cs": "phpcs -p",
"test": [
"composer validate --no-interaction",
"phpunit tests/php/*",
"phpcs -p"
]
}
}

1754
composer.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

0
language-data.json → data/language-data.json
View File

804
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

5
package.json
View File

@@ -34,8 +34,9 @@
"engine": {
"node": ">=6.0.0"
},
"license": "(MIT OR GPL-2.0+)",
"license": "GPL-2.0-or-later",
"scripts": {
"test": "mocha"
"test": "mocha tests/js/*.js",
"lint": "eslint src/*.js"
}
}

371
src/LanguageData.php Normal file
View File

@@ -0,0 +1,371 @@
<?php
/**
* Contains a utility class to query the language data.
*
* @file
* @license GPL-2.0-or-later
*/
namespace Wikimedia;
/**
* Utility class to query the language data.
*/
class LanguageData {
private static $instance;
public const OTHER_SCRIPT_GROUP = 'Other';
private const LANGUAGE_DATA_PATH = '../data/language-data.json';
private $data;
/**
* Returns an instance of the class
* @return LanguageData
*/
public static function get(): LanguageData {
if ( self::$instance === null ) {
self::$instance = new LanguageData();
self::$instance->loadData();
}
return self::$instance;
}
private function loadData() {
$this->data = json_decode( file_get_contents( __DIR__ . '/' . self::LANGUAGE_DATA_PATH ) );
}
/**
* Checks if a language code is valid
* @param string $languageCode
* @return bool
*/
public function isKnown( string $languageCode ): bool {
return isset( $this->data->languages->$languageCode );
}
/**
* Is this language a redirect to another language?
* @param string $languageCode Language code
* @return string|bool Target language code if it's a redirect or false if it's not
*/
public function isRedirect( string $languageCode ) {
if (
$this->isKnown( $languageCode ) &&
count( $this->getLanguage( $languageCode ) ) === 1
) {
return $this->getLanguage( $languageCode )[0];
}
return false;
}
/**
* Get all the languages
* @return object
*/
public function getLanguages() {
return $this->data->languages;
}
/**
* Returns the script of the language or false
* @param string $languageCode
* @return string|bool Language script if its a known language, else false.
*/
public function getScript( string $languageCode ) {
if ( !$this->isKnown( $languageCode ) ) {
return false;
}
$targetCode = $this->isRedirect( $languageCode );
if ( $targetCode ) {
return $this->getScript( $targetCode );
}
return $this->getLanguage( $languageCode )[0];
}
/**
* Returns the regions in which a language is spoken.
* @param string $languageCode
* @return string[]|bool Array of regions or false if language is unknown.
*/
public function getRegions( string $languageCode ) {
if ( !$this->isKnown( $languageCode ) ) {
return false;
}
$targetCode = $this->isRedirect( $languageCode );
if ( $targetCode ) {
return $this->getRegions( $targetCode );
}
return $this->getLanguage( $languageCode )[1];
}
/**
* Returns the autonym of the language.
* @param string $languageCode Language code
* @return string|bool
*/
public function getAutonym( $languageCode ) {
if ( !$this->isKnown( $languageCode ) ) {
return false;
}
$targetCode = $this->isRedirect( $languageCode );
if ( $targetCode ) {
return $this->getAutonym( $targetCode );
}
$language = $this->getLanguage( $languageCode );
return count( $language ) >= 2 ? $language[2] : $languageCode;
}
/**
* Returns all language codes and corresponding autonyms
* @return array
*/
public function getAutonyms(): array {
$languages = $this->getLanguages();
$languageAutonyms = [];
foreach ( $languages as $languageCode => $languageData ) {
if ( $this->isRedirect( $languageCode ) ) {
continue;
}
$languageAutonyms[$languageCode] = $this->getAutonym( $languageCode );
}
return $languageAutonyms;
}
/**
* Returns all languages written in the given scripts.
* @param string[] $scripts
* @return string[]
*/
public function getLanguagesInScripts( array $scripts ): array {
$languages = $this->getLanguages();
$languagesInScripts = [];
foreach ( $languages as $languageCode => $languageData ) {
if ( $this->isRedirect( $languageCode ) ) {
continue;
}
$script = $this->getScript( $languageCode );
if ( in_array( $script, $scripts ) ) {
$languagesInScripts[] = $languageCode;
}
}
return $languagesInScripts;
}
/**
* Returns all languages written in the given script.
* @param string $script
* @return string[]
*/
public function getLanguagesInScript( string $script ): array {
return $this->getLanguagesInScripts( [ $script ] );
}
/**
* Returns the script group of a script or 'Other' if it doesn't
* belong to any group.
* @param string $script Script code
* @return string script group name
*/
public function getGroupOfScript( string $script ): string {
$scriptGroups = $this->data->scriptgroups;
foreach ( $scriptGroups as $scriptGroup => $scriptGroupData ) {
if ( in_array( $script, $scriptGroupData ) ) {
return $scriptGroup;
}
}
return self::OTHER_SCRIPT_GROUP;
}
/**
* Returns the script group of a language.
* @param string $languageCode Language code
* @return string script group name
*/
public function getScriptGroupOfLanguage( string $languageCode ): string {
return $this->getGroupOfScript( $this->getScript( $languageCode ) );
}
/**
* Return the list of languages passed, grouped by script.
* @param string[] $languageCodes Array of language codes to group
* @return array Array of language codes grouped by script
*/
public function getLanguagesByScriptGroup( array $languageCodes ): array {
$languagesByScriptGroup = [];
foreach ( $languageCodes as $languageCode ) {
if ( !$this->isKnown( $languageCode ) ) {
continue;
}
$targetLanguageCode = $this->isRedirect( $languageCode );
if ( $targetLanguageCode === false ) {
$targetLanguageCode = $languageCode;
}
$langScriptGroup = $this->getScriptGroupOfLanguage( $targetLanguageCode );
if ( !isset( $languagesByScriptGroup[$langScriptGroup] ) ) {
$languagesByScriptGroup[$langScriptGroup] = [];
}
$languagesByScriptGroup[$langScriptGroup][] = $languageCode;
}
return $languagesByScriptGroup;
}
/**
* Returns an associative array of languages in several regions,
* grouped by script group.
* @param string[] $regions array of region codes
* @return array
*/
public function getLanguagesByScriptGroupInRegions( array $regions ): array {
$languagesByScriptGroupInRegions = [];
$languages = $this->getLanguages();
foreach ( $languages as $languageCode => $languageData ) {
if ( $this->isRedirect( $languageCode ) ) {
continue;
}
$languageRegions = $this->getRegions( $languageCode );
foreach ( $regions as $region ) {
if ( !in_array( $region, $languageRegions ) ) {
continue;
}
$langScriptGroup = $this->getScriptGroupOfLanguage( $languageCode );
if ( !isset( $languagesByScriptGroupInRegions[$langScriptGroup] ) ) {
$languagesByScriptGroupInRegions[$langScriptGroup] = [];
}
$languagesByScriptGroupInRegions[$langScriptGroup][] = $languageCode;
}
}
return $languagesByScriptGroupInRegions;
}
/**
* Returns an associative array of languages in a region, grouped by their script.
* @param string $region Region code
* @return array
*/
public function getLanguagesByScriptGroupInRegion( $region ): array {
return $this->getLanguagesByScriptGroupInRegions( [ $region ] );
}
/**
* Return the list of languages sorted by script groups.
* @param string[] $languageCodes Array of language codes to sort
* @return string[] Array of language codes
*/
public function sortByScriptGroup( array $languageCodes ) {
$groupedLanguageData = $this->getLanguagesByScriptGroup( $languageCodes );
ksort( $groupedLanguageData, SORT_STRING | SORT_FLAG_CASE );
$sortedLanguageData = [];
foreach ( $groupedLanguageData as $languageData ) {
$sortedLanguageData = array_merge( $sortedLanguageData, $languageData );
}
return $sortedLanguageData;
}
/**
* Sort languages by their autonym.
* @param string[] $languageCodes
* @return string[]
*/
public function sortByAutonym( array $languageCodes ): array {
$sortedLanguages = [];
foreach ( $languageCodes as $languageCode ) {
$autonym = $this->getAutonym( $languageCode );
if ( $autonym !== false ) {
$sortedLanguages[$languageCode] = $autonym;
}
}
asort( $sortedLanguages, SORT_STRING | SORT_FLAG_CASE );
return array_keys( $sortedLanguages );
}
/**
* Check if a language is right-to-left.
* @param string $languageCode Language code
* @return bool
*/
public function isRtl( string $languageCode ): bool {
$script = $this->getScript( $languageCode );
return in_array( $script, $this->data->rtlscripts );
}
/**
* Return the direction of the language. Returns false if the direction is unknown.
* @param string $languageCode Language code
* @return string|bool
*/
public function getDir( string $languageCode ) {
if ( $this->isKnown( $languageCode ) ) {
return $this->isRtl( $languageCode ) ? 'rtl' : 'ltr';
}
return false;
}
/**
* Returns the languages spoken in a territory.
* @param string $territory Territory code
* @return string[]|bool list of language codes
*/
public function getLanguagesInTerritory( string $territory ) {
if ( isset( $this->data->territories->$territory ) ) {
return $this->data->territories->$territory;
}
return false;
}
/**
* Adds a language in run time and sets its options as provided.
* If the target option is provided, the language is defined as a redirect.
* Other possible options are script, regions and autonym.
* @param string $languageCode New language code.
* @param array $options Language properties.
*/
public function addLanguage( string $languageCode, array $options ): void {
$languages = $this->getLanguages();
if ( isset( $options['target'] ) ) {
$languages->$languageCode = [ $options['target'] ];
} else {
$languages->$languageCode =
[ $options['script'], $options['regions'], $options['autonym'] ];
}
}
/**
* Return the language data based on language code. Performs no check, meant for
* internal use only.
* @param string $languageCode
* @return array
*/
private function getLanguage( string $languageCode ): array {
return $this->data->languages->$languageCode;
}
}

37
index.js → src/index.js
View File

@@ -1,4 +1,4 @@
var languageData = require( './language-data.json' );
var languageData = require( '../data/language-data.json' );
/**
* Utility functions for querying language data.
@@ -21,7 +21,8 @@ function isKnown( languageCode ) {
* @return {string} Target language code if it's a redirect or false if it's not
*/
function isRedirect( language ) {
return ( isKnown( language ) && languageData.languages[ language ].length === 1 ) ? languageData.languages[ language ][ 0 ] : false;
return ( isKnown( language ) && languageData.languages[ language ].length === 1 ) ?
languageData.languages[ language ][ 0 ] : false;
}
/**
@@ -77,7 +78,7 @@ function getAutonym( language ) {
/**
* Returns all language codes and corresponding autonyms
* @return {array}
* @return {Array}
*/
function getAutonyms() {
var language,
@@ -152,21 +153,21 @@ function getScriptGroupOfLanguage( language ) {
* @param {string[]} languages Array of language codes to group
* @return {string[]} Array of language codes
*/
function getLanguagesByScriptGroup( languages ) {
var languagesByScriptGroup = {},
languages, language, languageIndex, resolvedRedirect, langScriptGroup;
function getLanguagesByScriptGroup( languages ) {
var languagesByScriptGroup = {},
language, languageIndex, resolvedRedirect, langScriptGroup;
for ( languageIndex = 0; languageIndex < languages.length; languageIndex++ ) {
language = languages[ languageIndex ];
resolvedRedirect = isRedirect( language ) || language;
langScriptGroup = getScriptGroupOfLanguage( resolvedRedirect );
if ( !languagesByScriptGroup[ langScriptGroup ] ) {
languagesByScriptGroup[ langScriptGroup ] = [];
}
languagesByScriptGroup[ langScriptGroup ].push( language );
}
return languagesByScriptGroup;
}
for ( languageIndex = 0; languageIndex < languages.length; languageIndex++ ) {
language = languages[ languageIndex ];
resolvedRedirect = isRedirect( language ) || language;
langScriptGroup = getScriptGroupOfLanguage( resolvedRedirect );
if ( !languagesByScriptGroup[ langScriptGroup ] ) {
languagesByScriptGroup[ langScriptGroup ] = [];
}
languagesByScriptGroup[ langScriptGroup ].push( language );
}
return languagesByScriptGroup;
}
/**
* Returns an associative array of languages in several regions,
@@ -218,7 +219,7 @@ function sortByScriptGroup( languages ) {
scriptGroups = Object.keys( groupedLanguages ).sort();
for ( i = 0; i < scriptGroups.length; i++ ) {
allLanguages = allLanguages.concat( groupedLanguages[ scriptGroups[i] ] );
allLanguages = allLanguages.concat( groupedLanguages[ scriptGroups[ i ] ] );
}
return allLanguages;

0
data/spyc.php → src/util/spyc.php
View File

21
data/ulsdata2json.php → src/util/ulsdata2json.php
View File

@@ -6,26 +6,20 @@
* Niklas Laxström, Pau Giner, Santhosh Thottingal, Siebrand Mazeland and other
* contributors. See CREDITS for a list.
*
* UniversalLanguageSelector is dual licensed GPLv2 or later and MIT. You don't
* have to do anything special to choose one license or the other and you don't
* have to notify anyone which license you are using. You are free to use
* UniversalLanguageSelector in commercial projects as long as the copyright
* header is left intact. See files GPL-LICENSE and MIT-LICENSE for details.
*
* @file
* @ingroup Extensions
* @licence GNU General Public Licence 2.0 or later
* @licence MIT License
* @license GPL-2.0-or-later
*/
include __DIR__ . '/spyc.php';
print "Reading langdb.yaml...\n";
$yamlLangdb = file_get_contents( 'langdb.yaml' );
$yamlLangdb = file_get_contents( __DIR__ . '/../../data/langdb.yaml' );
$parsedLangdb = spyc_load( $yamlLangdb );
$supplementalDataFilename = 'supplementalData.xml';
$supplementalDataUrl =
// phpcs:ignore Generic.Files.LineLength
"https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/supplementalData.xml";
$curl = curl_init( $supplementalDataUrl );
@@ -51,13 +45,13 @@ if ( !( $supplementalData instanceof SimpleXMLElement ) ) {
}
print "CLDR supplemental data parsed successfully, reading territories info...\n";
$parsedLangdb['territories'] = array();
$parsedLangdb['territories'] = [];
foreach ( $supplementalData->territoryInfo->territory as $territoryRecord ) {
$territoryAtributes = $territoryRecord->attributes();
$territoryCodeAttr = $territoryAtributes['type'];
$territoryCode = (string) $territoryCodeAttr[0];
$parsedLangdb['territories'][$territoryCode] = array();
$territoryCode = (string)$territoryCodeAttr[0];
$parsedLangdb['territories'][$territoryCode] = [];
foreach ( $territoryRecord->languagePopulation as $languageRecord ) {
$languageAttributes = $languageRecord->attributes();
@@ -65,7 +59,7 @@ foreach ( $supplementalData->territoryInfo->territory as $territoryRecord ) {
// Lower case is a convention for language codes in ULS.
// '_' is used in CLDR for compound codes and it's replaced with '-' here.
$normalisedCode = strtr( strtolower( (string) $languageCodeAttr[0] ), '_', '-' );
$normalisedCode = strtr( strtolower( (string)$languageCodeAttr[0] ), '_', '-' );
$parsedLangdb['territories'][$territoryCode][] = $normalisedCode;
@@ -104,7 +98,6 @@ foreach ( $parsedLangdb['territories'] as $territoryCode => $languages ) {
$parsedLangdb['territories'][$territoryCode] =
array_unique( $parsedLangdb['territories'][$territoryCode] );
// We need to renumber or json conversion thinks these are objects
$parsedLangdb['territories'][$territoryCode] =
array_values( $parsedLangdb['territories'][$territoryCode] );

2
test/index.js → tests/js/index.js
View File

@@ -1,4 +1,4 @@
var languageData = require( '../index' ),
var languageData = require( __dirname + '/../../src/index' ),
assert = require( 'assert' );
describe( 'languagedata', function () {

275
tests/php/LanguageDataTest.php Normal file
View File

@@ -0,0 +1,275 @@
<?php
require __DIR__ . '/../../src/LanguageData.php';
use PHPUnit\Framework\TestCase;
use Wikimedia\LanguageData;
/**
* @coversDefaultClass \Wikimedia\LanguageData
*/
class LanguageDataTest extends TestCase {
/**
* @var LanguageData
*/
protected $languageData;
private const UNKNOWN_LANGUAGE_CODE = 'xyz';
protected function setUp(): void {
parent::setUp();
$this->languageData = LanguageData::get();
}
/**
* @covers isKnown
*/
public function testIsKnown() {
$this->assertTrue( $this->languageData->isKnown( 'en' ) );
$this->assertFalse( $this->languageData->isKnown( self::UNKNOWN_LANGUAGE_CODE ) );
}
/**
* @covers isRedirect
*/
public function testIsRedirect() {
$this->assertFalse( $this->languageData->isRedirect( 'en' ) );
$this->assertEquals( $this->languageData->isRedirect( 'aeb' ), 'aeb-arab' );
}
/**
* @covers getScript
*/
public function testGetScript() {
$this->assertEquals( $this->languageData->getScript( 'en' ), 'Latn' );
$this->assertFalse( $this->languageData->getScript( self::UNKNOWN_LANGUAGE_CODE ) );
}
/**
* @covers getRegions
*/
public function testGetRegions() {
$this->assertFalse( $this->languageData->getRegions( self::UNKNOWN_LANGUAGE_CODE ) );
$this->assertEquals( [ 'AF' ], $this->languageData->getRegions( 'aeb' ) );
$expected = [ 'EU', 'AM', 'AS' ];
$regions = $this->languageData->getRegions( 'en' );
foreach ( $expected as $region ) {
$this->assertContains( $region, $regions );
}
}
/**
* @covers getAutonym
*/
public function testGetAutonym() {
$this->assertFalse( $this->languageData->getAutonym( self::UNKNOWN_LANGUAGE_CODE ) );
$this->assertEquals(
'تونسي',
$this->languageData->getAutonym( 'aeb' ),
'Redirects return proper value in getAutonym.'
);
$this->assertEquals( 'English', $this->languageData->getAutonym( 'en' ) );
}
/**
* @covers getAutonyms
*/
public function testGetAutonyms() {
$autonyms = $this->languageData->getAutonyms();
$this->assertEquals( 'English', $autonyms['en'] );
$this->assertFalse(
isset( $autonyms['aeb'] ),
'Redirects are not present in getAutonyms.'
);
}
/**
* @covers getLanguagesInScripts
*/
public function testGetLanguagesInScripts() {
$this->assertEmpty(
$this->languageData->getLanguagesInScripts( [ self::UNKNOWN_LANGUAGE_CODE ] )
);
$expectedValues = $this->languageData->getLanguagesInScripts( [ 'Latn', 'Grek' ] );
$this->assertContains( 'zu', $expectedValues );
$this->assertContains( 'pnt', $expectedValues );
$this->assertNotContains(
'sr-el',
$expectedValues,
'Redirects are not present when fetching languages in scripts.'
);
}
/**
* @covers getGroupOfScript
*/
public function testGetGroupOfScript() {
$this->assertEquals( 'Latin', $this->languageData->getGroupOfScript( 'Latn' ) );
$this->assertEquals(
LanguageData::OTHER_SCRIPT_GROUP,
$this->languageData->getGroupOfScript( self::UNKNOWN_LANGUAGE_CODE )
);
}
/**
* @covers getScriptGroupOfLanguage
*/
public function testGetScriptGroupOfLanguage() {
$this->assertEquals(
LanguageData::OTHER_SCRIPT_GROUP,
$this->languageData->getScriptGroupOfLanguage( self::UNKNOWN_LANGUAGE_CODE )
);
$this->assertEquals(
'Latin',
$this->languageData->getScriptGroupOfLanguage( 'en' )
);
}
/**
* @covers getLanguagesByScriptGroup
*/
public function testGetLanguagesByScriptGroup() {
$actuals = $this->languageData->getLanguagesByScriptGroup( [ 'en', 'sr-el', 'tt-cyrl' ] );
$this->assertContains( 'tt-cyrl', $actuals['Cyrillic'] );
$this->assertContains( 'en', $actuals['Latin'] );
$this->assertContains( 'sr-el', $actuals['Latin'] );
}
/**
* @covers getLanguagesByScriptGroupInRegions
*/
public function testGetLanguagesByScriptGroupInRegions() {
$actuals = $this->languageData->getLanguagesByScriptGroupInRegions( [ 'AS', 'PA' ] );
$this->assertContains( 'tpi', $actuals['Latin'] );
$this->assertContains( 'ug-arab', $actuals['Arabic'] );
$this->assertContains( 'zh-sg', $actuals['CJK'] );
$this->assertNotContains(
'azb',
$actuals['Arabic'],
'Redirects are not present when languages grouped by script in a region.'
);
}
/**
* @covers sortByAutonym
*/
public function testSortByAutonym() {
$sorted = $this->languageData->sortByAutonym(
[
'atj', 'chr', 'chy',
'cr', 'en', 'es',
'fr', 'gn', 'haw',
'ike-cans', 'ik', 'kl',
'nl', 'pt', 'qu',
'srn', 'yi', self::UNKNOWN_LANGUAGE_CODE
]
);
$this->assertEquals(
[
'atj', 'gn', 'en',
'es', 'fr', 'haw',
'ik', 'kl', 'nl',
'pt', 'qu', 'srn',
'chy', 'yi', 'chr',
'ike-cans', 'cr'
],
$sorted
);
}
/**
* @covers sortByScriptGroup
*/
public function testSortByScriptGroup() {
$sorted = $this->languageData->sortByScriptGroup(
$this->languageData->sortByAutonym(
[
'atj', 'chr', 'chy',
'cr', 'en', 'es',
'fr', 'gn', 'haw',
'ike-cans', 'ik', 'kl',
'nl', 'pt', 'qu',
'srn', 'yi', self::UNKNOWN_LANGUAGE_CODE
]
)
);
$this->assertEquals(
[
'atj', 'gn', 'en',
'es', 'fr', 'haw',
'ik', 'kl', 'nl',
'pt', 'qu', 'srn',
'chy', 'yi', 'chr',
'ike-cans', 'cr'
],
$sorted
);
}
/**
* @covers isRtl
*/
public function testIsRtl() {
$this->assertFalse( $this->languageData->isRtl( 'en' ) );
$this->assertFalse( $this->languageData->isRtl( self::UNKNOWN_LANGUAGE_CODE ) );
$this->assertTrue( $this->languageData->isRtl( 'he' ) );
}
/**
* @covers getDir
*/
public function testGetDir() {
$this->assertEquals( 'ltr', $this->languageData->getDir( 'en' ) );
$this->assertEquals( 'rtl', $this->languageData->getDir( 'he' ) );
$this->assertFalse( $this->languageData->getDir( self::UNKNOWN_LANGUAGE_CODE ) );
}
/**
* @covers getLanguagesInTerritory
*/
public function testGetLanguagesInTerritory() {
$actualsAFG = $this->languageData->getLanguagesInTerritory( 'AF' );
$actualsAT = $this->languageData->getLanguagesInTerritory( 'AT' );
$this->assertContains( 'de', $actualsAT );
$this->assertContains( 'bar', $actualsAT );
$this->assertNotContains( 'he', $actualsAT );
$this->assertContains( 'ug-arab', $actualsAFG );
$this->assertContains( 'tk', $actualsAFG );
$this->assertNotContains( 'de', $actualsAFG );
}
/**
* @covers addLanguage
*/
public function testAddLanguage() {
$this->assertFalse( $this->languageData->isKnown( 'xyz' ) );
$this->assertNotContains(
'xyz',
$this->languageData->getLanguagesByScriptGroupInRegion( 'AF' )['Latin']
);
$this->languageData->addLanguage( self::UNKNOWN_LANGUAGE_CODE, [
'script' => "Latn",
'regions' => [
"AF"
],
'autonym' => "Test Language"
] );
$this->assertTrue( $this->languageData->isKnown( self::UNKNOWN_LANGUAGE_CODE ) );
$this->assertContains(
self::UNKNOWN_LANGUAGE_CODE,
$this->languageData->getLanguagesByScriptGroupInRegion( 'AF' )['Latin']
);
}
}