Improve ULS language search api

* Store prefixes and infixes separately in the data
* First match language code, then prefixes, then infixes
* Try to use suggestion either in user language or autonym first
* use formatversion=2 to avoid escaping Unicode

Using Language::fetchLanguageName might can have a small
performance impact. On the other hand there is now check
to skip languages we already found, avoiding some fuzzy
matching.

This is in a preparation for a change in jquery.uls to use
the search API more, while trying to reduce the amount of
weird autocompletion suggestions we show to the user.

Bug: T73891
Change-Id: Id94c5352d9a591969bf90144d1d2d5e758d08301

api/ApiLanguageSearch.php

@@ -22,12 +22,11 @@
  * @ingroup API
  */
 class ApiLanguageSearch extends ApiBase {
-
 	public function execute() {
 		$params = $this->extractRequestParams();
 		$search = $params['search'];
 		$typos = $params['typos'];
-		$searches = LanguageNameSearch::search( $search, $typos );
+		$searches = LanguageNameSearch::search( $search, $typos, $this->getLanguage()->getCode() );
 		$result = $this->getResult();
 		$result->addValue( null, $this->getModuleName(), $searches );
 	}

data/LanguageNameIndexer.php

@@ -58,11 +58,13 @@ class LanguageNameIndexer extends Maintenance {
 				foreach ( $words as $index => $word ) {
 					$bucket = LanguageNameSearch::getIndex( $word );
 
+					$type = 'prefix';
 					$display = $translation;
 					if ( $index > 0 && count( $words ) > 1 ) {
+						$type = 'infix';
 						$display = "$word <$translation>";
 					}
-					$buckets[$bucket][$display] = $targetLanguage;
+					$buckets[$bucket][$type][$display] = $targetLanguage;
 				}
 			}
 		}
@@ -87,10 +89,25 @@ class LanguageNameIndexer extends Maintenance {
 
 		foreach ( $specialLanguages as $targetLanguage => $translation ) {
 			$bucket = LanguageNameSearch::getIndex( $translation );
-			$buckets[$bucket][$translation] = $targetLanguage;
+			$buckets[$bucket]['prefix'][$translation] = $targetLanguage;
+		}
+
+		$lengths = [];
+		// Sorting the bucket contents gives two benefits:
+		// - more consistent output across environments
+		// - shortest matches appear first, especially exact matches
+		// Sort buckets by index
+		ksort( $buckets );
+		foreach ( $buckets as $index => &$bucketTypes ) {
+			$lengths[] = array_sum( array_map( 'count', $bucketTypes ) );
+			// Ensure 'prefix' is before 'infix';
+			krsort( $bucketTypes );
+			// Ensure each bucket has entries sorted
+			foreach ( $bucketTypes as $type => &$bucket ) {
+				ksort( $bucket );
+			}
 		}
 
-		$lengths = array_values( array_map( 'count', $buckets ) );
 		$count = count( $buckets );
 		$min = min( $lengths );
 		$max = max( $lengths );
@@ -113,7 +130,6 @@ class LanguageNameSearchData {
 
 PHP;
 
-		ksort( $buckets );
 		// Format for short array format
 		$data = var_export( $buckets, true );
 		$data = str_replace( "array (", '[', $data );

data/LanguageNameSearch.php

@@ -18,29 +18,91 @@
  * @licence MIT License
  */
 class LanguageNameSearch {
-	public static function search( $searchKey, $typos = 0 ) {
+	/**
+	 * Find languages with fuzzy matching.
+	 * The order of results is following:
+	 * 1: exact language code match
+	 * 2: exact language name match in any language
+	 * 3: prefix language name match in any language
+	 * 4: infix language name match in any language
+	 *
+	 * The returned language name for autocompletion is the first one that
+	 * matches in this list:
+	 * 1: exact match in [user, autonym, any other language]
+	 * 2: prefix match in [user, autonum, any other language]
+	 * 3: inline match in [user, autonym, any other language]
+	 *
+	 * @param string $searchKey
+	 * @param int $typos
+	 * @param string $userLanguage Language tag.
+	 * @return array
+	 */
+	public static function search( $searchKey, $typos = 0, $userLanguage = null ) {
+		$results = [];
 		$searchKey = mb_strtolower( $searchKey );
-		$index = self::getIndex( $searchKey );
 
-		if ( !isset( LanguageNameSearchData::$buckets[$index] ) ) {
-			return [];
+		// Always prefer exact language code match
+		if ( Language::isKnownLanguageTag( $searchKey ) ) {
+			$name = mb_strtolower( Language::fetchLanguageName( $searchKey, $userLanguage ) );
+			// Check if language code is a prefix of the name
+			if ( strpos( $name, $searchKey ) === 0 ) {
+				$results[$searchKey] = $name;
+			} else {
+				$results[$searchKey] = "$searchKey <$name>";
+			}
 		}
 
-		$bucket = LanguageNameSearchData::$buckets[$index];
+		$index = self::getIndex( $searchKey );
+		$bucketsForIndex = [];
+		if ( isset( LanguageNameSearchData::$buckets[$index] ) ) {
+			$bucketsForIndex = LanguageNameSearchData::$buckets[$index];
+		}
 
-		$results = [];
+		// types are 'prefix', 'infix' (in this order!)
+		foreach ( $bucketsForIndex as $bucketType => $bucket ) {
 			foreach ( $bucket as $name => $code ) {
-			// Prefix search
-			if ( strrpos( $name, $searchKey, -strlen( $name ) ) !== false
-				|| ( $typos > 0 && self::levenshteinDistance( $name, $searchKey ) <= $typos )
-			) {
-				$results[$code] = $name;
+				// We can skip checking languages we already have in the list
+				if ( isset( $results[ $code ] ) ) {
+					continue;
+				}
+
+				// Apply fuzzy search
+				if ( !self::matchNames( $name, $searchKey, $typos ) ) {
+					continue;
+				}
+
+				// Once we find a match, figure out the best name to display to the user
+				// If $userLanguage is not provided (null), it is the same as autonym
+				$candidates = [
+					mb_strtolower( Language::fetchLanguageName( $code, $userLanguage ) ),
+					mb_strtolower( Language::fetchLanguageName( $code, null ) ),
+					$name
+				];
+
+				foreach ( $candidates as $candidate ) {
+					if ( $searchKey === $candidate ) {
+						$results[$code] = $candidate;
+						continue 2;
+					}
+				}
+
+				foreach ( $candidates as $candidate ) {
+					if ( self::matchNames( $candidate, $searchKey, $typos ) ) {
+						$results[$code] = $candidate;
+						continue 2;
+					}
+				}
 			}
 		}
 
 		return $results;
 	}
 
+	public static function matchNames( $name, $searchKey, $typos ) {
+		return strrpos( $name, $searchKey, -strlen( $name ) ) !== false
+			|| ( $typos > 0 && self::levenshteinDistance( $name, $searchKey ) <= $typos );
+	}
+
 	public static function getIndex( $name ) {
 		$codepoint = self::getCodepoint( $name );
 

resources/js/ext.uls.mediawiki.js

@@ -23,7 +23,7 @@
 	// MediaWiki overrides for ULS defaults
 	$.fn.uls.defaults = $.extend( $.fn.uls.defaults, {
 		languages: mw.config.get( 'wgULSLanguages' ) || {},
-		searchAPI: mw.util.wikiScript( 'api' ) + '?action=languagesearch&format=json'
+		searchAPI: mw.util.wikiScript( 'api' ) + '?action=languagesearch&format=json&formatversion=2'
 	} );
 
 	// No need of IME in the ULS language search bar

tests/phpunit/LanguageSearchTest.php

@@ -23,8 +23,11 @@ class LanguageSearchTest extends PHPUnit_Framework_TestCase {
 	 * @dataProvider searchDataProvider
 	 */
 	public function testSearch( $searchKey, $expected ) {
-		$actual = LanguageNameSearch::search( $searchKey, 1 );
+		$actual = LanguageNameSearch::search( $searchKey, 1, 'en' );
+		// This is for better error messages
 		$this->assertEquals( $expected, $actual );
+		// This is for identical order
+		$this->assertSame( $expected, $actual );
 	}
 
 	public function searchDataProvider() {
@@ -34,8 +37,8 @@ class LanguageSearchTest extends PHPUnit_Framework_TestCase {
 			]
 			],
 			[ 'മല', [
-				'ml' => 'മലയാളം',
 				'mg' => 'മലഗാസി',
+				'ml' => 'മലയാളം',
 				'ms' => 'മലെയ്',
 			]
 			],
@@ -43,15 +46,16 @@ class LanguageSearchTest extends PHPUnit_Framework_TestCase {
 				'fi' => 'φινλανδικά',
 			]
 			],
-			[ 'blah', []
+			[ 'blargh', []
 			],
 			[ 'الفرنسية', [
 				'fr' => 'الفرنسية',
-				'fr-ca' => 'الفرنسية الكندية',
 				'fr-ch' => 'الفرنسية السويسرية',
-				'frm' => 'الفرنسية الوسطى',
 				'fro' => 'الفرنسية القديمة',
-				'crs' => 'الفرنسية الكريولية السيشيلية'
+				'frc' => 'الفرنسية الكاجونية',
+				'crs' => 'الفرنسية الكريولية السيشيلية',
+				'fr-ca' => 'الفرنسية الكندية',
+				'frm' => 'الفرنسية الوسطى',
 			]
 			],
 			[ 'മലയളം', [
@@ -59,7 +63,7 @@ class LanguageSearchTest extends PHPUnit_Framework_TestCase {
 			]
 			],
 			[ 'punja', [
-				'pa' => 'punjabi <èdè punjabi>',
+				'pa' => class_exists( 'LanguageNames' ) ? 'punjabi' : 'punjaabi sennii',
 				'pnb' => 'punjabi western',
 			]
 			],
@@ -72,22 +76,22 @@ class LanguageSearchTest extends PHPUnit_Framework_TestCase {
 			]
 			],
 			[ 'chinese', [
-				'zh-hans' => 'chinese simplified',
-				'zh-hant' => 'chinese traditional',
-				'zh' => 'chinesesch',
+				'zh' => 'chinese',
 				'zh-cn' => 'chinese (china)',
 				'zh-hk' => 'chinese (hong kong)',
 				'zh-min-nan' => 'chinese (min nan)',
 				'zh-sg' => 'chinese (singapore)',
 				'zh-tw' => 'chinese (taiwan)',
-				'cdo' => 'chinese <min dong chinese>',
-				'gan' => 'chinese <isi-gan chinese>',
-				'hak' => 'chinese <isi-hakka chinese>',
-				'lzh' => 'chinesesch <klassescht chinesesch>',
+				'zh-hans' => 'chinese simplified',
+				'zh-hant' => 'chinese traditional',
+				'zh-classical' => 'chinese <classical chinese>',
+				'gan' => 'chinese <gan chinese>',
+				'hak' => 'chinese <hakka chinese>',
 				'nan' => 'chinese <isi-min nan chinese>',
 				'wuu' => 'chinese <isi-wu chinese>',
-				'zh-classical' => 'chinese <classical chinese>',
 				'hsn' => 'chinese <isi-xiang chinese>',
+				'lzh' => 'chinese <literary chinese>',
+				'cdo' => 'chinese <min dong chinese>',
 			]
 			],
 			[ 'finish', [