Module:User:IsomorphycSandbox/data tables

This module lacks a documentation subpage. You may create it.

Useful links: root page • root page’s subpages • links • transclusions • testcases • user page • user talk page • userspace

This is a private module sandbox of IsomorphycSandbox, for their own experimentation. Items in this module may be added and removed at IsomorphycSandbox's discretion; do not rely on this module's stability.

---

local export = {}

local base_location = "Module:data tables"
local metadata = mw.loadData(base_location.."/metadata")
local metadata = { ["shard_location_base"]={"Module:data tables/dataC"},["shards"]={100},["tables"] = {"grc_RWoodhouse_lemma_to_infinitives", "grc_RLBG_lemma_to_index", "grc_RWoodhouse_lemma_to_headwords", "freezer", "la_RMA_index_to_phrases", "grc_RCunliffe_lemma_to_index", "la_RMA_lemmas_no_collision_to_ix_phrase"}} 
local shard_location_base = metadata["shard_location_base"][1] --"Module:data tables/data"
local shards = metadata["shards"][1]
local tables = metadata["tables"] --{"grc_RLBG_lemma_to_index", "grc_RWoodhouse_lemma_to_headwords", "grc_RWoodhouse_lemma_to_infinitives", "la_RMA_index_to_phrases", "grc_RCunliffe_lemma_to_index", "la_RMA_lemmas_no_collision_to_ix_phrase" }

function load_tables_unsharded()
	function maybe_mw_load(x)
		local success, val = pcall(function() return mw.loadData(x) end)
 		if success then return val else return nil end	end
	local unsharded, i = {}, 0
    while #unsharded == i do
      local val = maybe_mw_load(shard_location_base.."Unsharded"..tostring(i))  ---shard_location_base
      if val ~= nil then 
      	unsharded[#unsharded+1] = val end
      i = i + 1
    end
    return unsharded
end

local unsharded = load_tables_unsharded()

function map(f, xs)
	local ys = {}	
	for i,x in pairs(xs) do ys[i]=f(x) end 
	return ys
end

function map_string(f, s)
	local ys = {}	
	for i = 1,#s do ys[i]=f(s:sub(i,i)) end 
	return ys
end

function fold(f, l, xs)
	for i,x in pairs(xs) do l = f(l, x) end
	return l
end

function hash(s)
	return fold(function(x,y) return x + y end, 0, map_string(function(x) return x:byte() end, s))%shards
end

function export.list_tables()
	return map(function (x) return x end, tables)
end

function export.load_table(n)
	local t = {}
	for i = 1,shards do 
		local curr = mw.loadData(shard_location_base..i)
		if curr[n] ~= nil then
			for k,v in pairs(curr[n]) do
				t[k]=v end end end
	for i,curr in pairs(unsharded) do 
		if curr[n] ~= nil then
			for k,v in pairs(curr[n]) do
				t[k]=v end end end
	return t
end

-- eventually: 
--   1 : loop through all unsharded if multiple
--   2 : index k in unsharded and shard separately, returning from unsharded preferentially
function export.index_table(t, k)
    local shard = mw.loadData(shard_location_base..hash(k))				
    if unsharded[1] ~= nil and unsharded[1][t] ~= nil and unsharded[1][t][k] ~= nil then return unsharded[1][t][k] else if shard[t] ~= nil then return shard[t][k] else return nil end end
end

function export.index_table_all(t, ks)
    baskets, result = {}, {}
    for i,k in pairs(ks) do 
    	h = hash(k)
    	if baskets[h] == nil then 
    		baskets[h] = {k} 
    	else 
    		baskets[h][#(baskets[h])+1] = k
    	end
    end
    for h,b in pairs(baskets) do 
    	local shard = mw.loadData(shard_location_base..h)
    	for i,k in pairs(b) do
    		if unsharded ~= nil and unsharded[1] ~= nil and unsharded[t] ~= nil then 		--here, loop over all unsharded if multiple
    			result[k] = unshared[t][k]
			else 
				if shard[t][k] ~= nil then 
					result[k] = shard[t][k]
    			end
			end
		end
	end
    return result
end

function export.select(t,p)
	return nil  --Not implemented.
end

return export