From Wiktionary, the free dictionary
Jump to navigation Jump to search

This module will sort text in the Han script. It is used to sort Ai-Cham, Southern Amami Ōshima, Central Bai, Panyi Bai, Southern Bai, Biao-Jiao Mien, Biyo, Eastern Min, Jin, Mandarin, Northern Pinghua, Chinese Pidgin English, Puxian Min, Macau Pidgin Portuguese, Southern Pinghua, Huizhou, Central Min, Dungan, Daur, E, Gan, Hakka, Yemaek, Xiang, Japanese, Hachijō, Kikai, Lama Bai, Middle Chinese, Literary Chinese, Jie, Middle Vietnamese, Caolan, Northern Min, Translingual, Miyako, Min Nan, Datian Min, Hokkien, Hailufeng Min, Hainanese, Longyan Min, Leizhou Min, Teochew, Zhenan Min, Sanxiang Min, Nùng, Old Chinese, Old Japanese, Okinoerabu, Old Uyghur, Bouyei, Baekje, Rouran, Tuyuhun, Tuoba, Wuhuan, Xianbei, Northern Amami Ōshima, Yaeyama, Okinawan, Sui, Bailang, Tokunoshima, Alchuka, Bala, Kyakala, Tày, Vietnamese, Wu, Waxiang, Classical Tibetan, Middle Mongol, Buyeo, Kunigami, Yonaguni, Yoron, Cantonese, Zhuang, Zauzou, Chinese, Shaozhou Tuhua, Sichuanese, Taishanese, Goguryeo, Zakhring, Khitan, and Gaya. The module should preferably not be called directly from templates or other modules. To use it from a template, use {{sortkey}}. Within a module, use Module:languages#Language:makeSortKey.

For testcases, see Module:Hani-sortkey/testcases.


makeSortKey(text, lang, sc)
Generates a sortkey for a given piece of text written in the script specified by the code sc, and language specified by the code lang.
When the sort fails, returns nil.

The demonstration functions that generated the content shown below are housed in Module:Hani-sortkey/templates. Modifications to the module can be tested in Module:Hani-sortkey/sandbox. Sortkeys for individual characters are retrieved from one of 178 data modules. Module:Hani-sortkey/data creates documentation for these modules.

Show sortkeys

  • (乙01)
  • 𡆔 (口23)
  • 𡎇 (土09)
  • 阿坝 (阜05土04)

Ideographic description sequences

local export = {}

local m_str_utils = require("Module:string utilities")

local byte = string.byte
local codepoint = m_str_utils.codepoint
local concat = table.concat
local convert_iteration_marks = require("Module:Hani").convert_iteration_marks
local explode = m_str_utils.explode_utf8
local format = string.format
local gmatch = string.gmatch
local gsub = string.gsub
local insert = table.insert
local sub = string.sub
local u = m_str_utils.char
local ugsub = mw.ustring.gsub
local umatch = mw.ustring.match
local upper = m_str_utils.upper

local m_data = require("Module:Hani-sortkey/data/serialized")
local m_data_core = mw.loadData("Module:Hani-sortkey/data/core")
local cache = {}

	Returns the index in the string where the ideographic description sequence
	(IDS) ends, or the index of the end of the string. Iterates whenever
	another ideographic description character (IDC) is found.
local function findEndOfIDS(text, IDchar, i)
	if not (text and IDchar and i) then
		return nil
	local j = i
	local component = 1
	-- Number of components expected after current IDC.
	local components = m_data_core.ids[IDchar]
	while component <= components do
		j = j + 1
		local char = text[j]
		if not char then
		elseif m_data_core.ids[char] then
			j = findEndOfIDS(text, char, j)
		component = component + 1
		If the expected number of components has been found,
		return the current index in the text.
	if component - components == 1 then
		return j
		return nil

local function unserialize(a, b)
	return m_data_core.radicals[byte(a)] .. format("%02d", byte(b) - 10)

-- The data is stored in [[Module:Hani-sortkey/data]]. This data is not accessed directly (due to the large amount of memory this would consume), but is instead stored in a serialized form as [[Module:Hani-sortkey/data/serialized]]. If the data is changed, the new serialized data can be generated with [[Module:Hani-sortkey/data/serializer]].
function export.getData(char)
	if type(char) == "string" then
		char = codepoint(char)
	elseif type(char) ~= "number" then
		error("getData must operate on a single character or codepoint.")
	local offset, s, f, lookup = 0
	for i = 2, m_data_core.ranges.n, 2 do
		s, f = m_data_core.ranges[i - 1], m_data_core.ranges[i]
		if char > f then
			offset = offset + f - s + 1
		elseif char >= s and char <= f then
			lookup = 2 * (offset + char - s + 1)
			return (gsub(sub(m_data, lookup - 1, lookup), "(.)(.)", unserialize))
	return u(char)

function export.makeSortKey(text, lang, sc)
	local scripts = {
		Hani = true,
		Hans = true,
		Hant = true,
		Jpan = true,
		Kore = true
	if sc and not scripts[sc] then
		return upper(text)
-- Convert any iteration marks into full characters, and remove any spaces. Also remove punctuation if the term contains non-punctuation (so that entries for punctuation characters can still be sorted properly).
	text = ugsub(convert_iteration_marks(text), "%s+", "")
	if not umatch(text, "^%p+$") then
		text = ugsub(text, "%p+", "")
	text = explode(text)
	local sort, text_len, i = {}, #text, 0
	while i < text_len do
		i = i + 1
		local char = text[i]
		if m_data_core.preconvert[char] then
			local j = 0
			for c in gmatch(m_data_core.preconvert[char], ".[\128-\191]*") do
				if j == 0 then
					text[i] = c
					insert(text, i + j, c)
				j = j + 1
			char = text[i]
			text_len = #text
			If we encounter an ideographic description character (IDC),
			find out if it begins a valid ideographic description sequence (IDS).
			If the IDS is valid and a sortkey for it is listed in
			[[Module:Hani-sortkey/data/unsupported]], then return
			the sortkey, and move to the next character after the
			Otherwise, insert the IDC into the sortkey and move to the next
			character after the IDC.
			If the IDS is valid and no sortkey for it is found, track it.
		if m_data_core.ids[char] then
			local j = findEndOfIDS(text, char, i)
			local IDS, data
			if j then
				IDS = concat(text, nil, i, j)
				data = m_data_core.unsupported[IDS]
			if not data then
				if IDS then
					mw.log("ideographic description sequence without sortkey: '"
						.. IDS .. "'")
					mw.log("invalid ideographic description sequence at the beginning of '"
						.. text[i] .. "'")
			if IDS and data then
				insert(sort, data)
				i = j
				insert(sort, char)
			if not cache[char] then
				cache[char] = export.getData(char)
			insert(sort, cache[char])
	return concat(sort)

return export