Convert telex to store mesages in a more scalable way.

I was severely worried about `mbox` and `spool` becoming way
too large way too quick - We just can't store the entire array
of full messages in there all the time.

Instead, both those arrays become simple string arrays of `msgid`
objects, which are nothing more but simple numeric strings. Each
message can now live in mod_storage which should scale a lot better
and make player mboxes a lot smaller - it can now easily hold a few
hundred messages without growing much.

This means all the messages are in mod_storage. We can at a later
point perhaps consider compression.

There's compat code to read old mbox formats, but not spool. Thus
the server spool must be empty when this code is deployed.

mods/telex/init.lua

@@ -42,8 +42,7 @@ telex = {}
   - reply to a message
   - delete a message
 
-  message table format:
+  msg = {
-  {
     "from" = <string>,
     "to" = <string>,
     "subject" = <string>,
@@ -52,6 +51,18 @@ telex = {}
     "age" = <int>
   }
 
+  mbox = {
+    [1] = msgid,
+    [2] = msgid,
+    ...
+  }
+
+  spool = {
+    [1] = msgid,
+    [2] = msgid,
+    ...
+  }
+
   Spool/mbox format: array of messages
 
 ]]--
@@ -59,12 +70,45 @@ telex = {}
 local S = minetest.get_mod_storage()
 assert(S)
 
+-- helper functions for handling msgid
+
+-- allocate a new msgid str
+function telex.msgid()
+	local msgid = S:get_int("msgid") + 1
+	S:set_int("msgid", msgid)
+	return "m" .. tostring(msgid)
+end
+
+-- get
+function telex.get_msg(msgid)
+	if not msgid then
+		return nil
+	end
+	local msg = S:get_string(msgid)
+	if msg then
+		return telex.decode(msg)
+	else
+		return nil
+	end
+end
+
+-- set
+function telex.save_msg(msgid, msg)
+	S:set_string(msgid, telex.encode(msg))
+end
+
+-- remove
+function telex.remove_msg(msgid)
+	S:set_string(msgid, "")
+end
+
 -- returns an array of strings
 function telex.list(player)
 	local pmeta = player:get_meta()
 	local mbox = telex.decode(pmeta:get_string("telex_mbox"))
 	local list = {}
-	for n, msg in pairs(mbox) do
+	for n, msgid in pairs(mbox) do
+		local msg = telex.get_msg(msgid)
 		local unread = "!"
 		if msg.read == 1 then
 			unread = " "
@@ -81,7 +125,10 @@ function telex.get(player, no)
 	if not mbox then
 		return { "You have no messages." }
 	end
-	local msg = mbox[no]
+
+	local msgid = mbox[no]
+	local msg = telex.get_msg(msgid)
+
 	if not msg then
 		return { "No such message exists." }
 	end
@@ -96,7 +143,8 @@ function telex.read(player, no)
 	if not mbox then
 		return { "You have no messages." }
 	end
-	local msg = mbox[no]
+	local msgid = mbox[no]
+	local msg = telex.get_msg(msgid)
 	if not msg then
 		return { "No such message exists." }
 	end
@@ -112,7 +160,8 @@ function telex.read(player, no)
 
 	-- mark as read and store
 	msg.read = 1
-	mbox[no] = msg
+	telex.save_msg(msgid, msg)
+	mbox[no] = msgid
 	pmeta:set_string("telex_mbox", telex.encode(mbox))
 
 	return list
@@ -125,68 +174,94 @@ function telex.delete(player, no)
 	if not mbox then
 		return { "You have no messages." }
 	end
-	local msg = mbox[no]
+	local msgid = mbox[no]
+	local msg = telex.get_msg(msgid)
 	if not msg then
 		return { "No such message exists." }
 	end
 
 	table.remove(mbox, no)
+	telex.remove_msg(msgid)
 	pmeta:set_string("telex_mbox", telex.encode(mbox))
 
 	return { "Message deleted." }
 end
 
--- returns nothing
+-- external should use `send`, internal uses `deliver`
-function telex.deliver(message)
+function telex.send(msg)
-	assert(message.from)
+	assert(msg.from)
-	assert(message.to)
+	assert(msg.to)
-	assert(message.subject)
+	assert(msg.subject)
-	assert(message.content)
+	assert(msg.content)
 
-	local player = minetest.get_player_by_name(message.to)
+	local msgid = telex.msgid()
+	telex.save_msg(msgid, msg)
+	telex.deliver(msgid, msg)
+end
+
+-- returns nothing
+function telex.deliver(msgid, msg)
+	-- msg is optional
+	if not msg then
+		msg = telex.get_msg(msgid)
+	end
+	local player = minetest.get_player_by_name(msg.to)
 	if player then
+		-- remove age, no longer needed
+		msg.age = nil
+		telex.save_msg(msgid, msg)
 		-- retrieve inbox
 		local pmeta = player:get_meta()
 		local mbox = telex.decode(pmeta:get_string("telex_mbox"))
-		table.insert(mbox, message)
+		table.insert(mbox, msgid)
 		pmeta:set_string("telex_mbox", telex.encode(mbox))
-		minetest.chat_send_player(message.to, "You have a new message from <" ..
+		minetest.chat_send_player(msg.to, "You have a new message from <" ..
-				message.from .. ">, use a terminal to read your messages");
+				msg.from .. ">, use a terminal to read your messages");
-		minetest.log("action", "delivered a message from <" .. message.from .. "> to <" .. message.to .. ">")
+		minetest.log("action", "delivered a message from <" .. msg.from .. "> to <" .. msg.to .. ">")
 	else
 		-- append to spool
-		message.age = 7 * 86400 -- 7 days max in spool
+		msg.age = 7 * 86400 -- 7 days max in spool
+		telex.save_msg(msgid, msg)
 		local spool = telex.decode(S:get_string("telex_spool"))
-		table.insert(spool, message)
+		table.insert(spool, msgid)
 		S:set_string("telex_spool", telex.encode(spool))
-		minetest.log("action", "spooled a message from <" .. message.from .. "> to <" .. message.to .. ">")
+		minetest.log("action", "spooled a message from <" .. msg.from .. "> to <" .. msg.to .. ">")
-		announce.admins("spooled a message from <" .. message.from .. "> to <" .. message.to .. ">")
+		announce.admins("spooled a message from <" .. msg.from .. "> to <" .. msg.to .. ">")
 	end
 end
 
 -- returns nothing
-function telex.retour(message)
+function telex.retour(msgid)
-	if message.from == "MAILER-DEAMON" then
+	local msg = telex.get_msg(msgid)
+	if msg.from == "MAILER-DEAMON" then
 		-- discard, we tried hard enough!
-		minetest.log("action", "discarded an expired spool message from <" .. message.from .. "> to <" .. message.to .. ">")
+		minetest.log("action", "discarded an expired spool message from <" .. msg.from .. "> to <" .. msg.to .. ">")
-		announce.admins("discarded an expired spool message from <" .. message.from .. "> to <" .. message.to .. ">")
+		announce.admins("discarded an expired spool message from <" .. msg.from .. "> to <" .. msg.to .. ">")
 		return
 	end
 
-	local to = message.to
+	local to = msg.to
-	local from = message.from
+	local from = msg.from
-	message.to = from
+	msg.to = from
-	message.from = "MAILER-DAEMON"
+	msg.from = "MAILER-DAEMON"
-	message.subject = "UNDELIVERABLE: " .. message.subject
+	msg.subject = "UNDELIVERABLE: " .. msg.subject
-	message.age = 86400 * 30 -- return mail for 30 days max, then discard.
+	msg.age = 86400 * 30 -- return mail for 30 days max, then discard.
-	table.insert(message.content, 1, "Your message to <" .. to .. "> was unable to be delivered.")
+	table.insert(msg.content, 1, "Your message to <" .. to .. "> was unable to be delivered.")
-	table.insert(message.content, 2, "================== ORIGINAL MESSAGE BELOW ================")
+	table.insert(msg.content, 2, "================== ORIGINAL MESSAGE BELOW ================")
-	telex.deliver(message)
+
+	-- remove the old msg, it's no longer in spool
+	telex.remove_msg(msgid)
+
+	-- allocate a new ID for the return msg
+	local msgid2 = telex.msgid()
+	telex.save_msg(msgid2, msg)
+	telex.deliver(msgid2, msg)
+
 	minetest.log("action", "returned a message from <" .. from .. "> back to <" .. to .. ">")
 	announce.admins("returned a message from <" .. from .. "> back to <" .. to .. ">")
 end
 
--- returns message/mbox
+-- returns message/mbox --FIXME compress/decompress
 function telex.decode(digest)
 	if not digest or digest == "" then
 		return {}
@@ -205,6 +280,22 @@ function telex.encode(message_or_mbox)
 end
 
 minetest.register_on_joinplayer(function(player)
+	-- convert old mbox
+	local pmeta = player:get_meta()
+	local mbox = telex.decode(pmeta:get_string("telex_mbox"))
+	local save = false
+	for k, v in pairs(mbox) do
+		if type(v) == "table" then
+			local msgid = telex.msgid()
+			telex.save_msg(msgid, v)
+			mbox[k] = msgid
+			save = true
+		end
+	end
+	if save then
+		pmeta:set_string("telex_mbox", telex.encode(mbox))
+	end
+
 	-- check the spool for messages for `player`
 	local spool = telex.decode(S:get_string("telex_spool"))
 	local name = player:get_player_name()
@@ -212,10 +303,11 @@ minetest.register_on_joinplayer(function(player)
 	-- deliver items for this player
 	local nos = {}
 	local del = {}
-	for k, msg in pairs(spool) do
+	for k, msgid in pairs(spool) do
+		local msg = telex.get_msg(msgid)
 		if msg.to == name then
 			table.insert(nos, k)
-			table.insert(del, msg)
+			table.insert(del, msgid)
 		end
 	end
 
@@ -226,14 +318,13 @@ minetest.register_on_joinplayer(function(player)
 	S:set_string("telex_spool", telex.encode(spool))
 
 	-- now deliver them to player
-	for _, msg in pairs(del) do
+	for _, msgid in pairs(del) do
-		telex.deliver(msg)
+		telex.deliver(msgid)
 	end
 
 	if #nos > 0 then
 		--FIXME minetest.after()
 		minetest.chat_send_player(name, "You have " .. #nos .. " new message(s), use a terminal to read your messages");
-
 	end
 end)
 
@@ -241,12 +332,14 @@ function telex.process_spool()
 	local spool = telex.decode(S:get_string("telex_spool"))
 	local old = {}
 	local del = {}
-	for k, msg in pairs(spool) do --FIXME BAD
+	for k, msgid in pairs(spool) do
+		local msg = telex.get_msg(msgid)
 		msg.age = msg.age - 3600
 		if msg.age < 0 then
 			old[#old + 1] = k
-			table.insert(del, msg)
+			table.insert(del, msgid)
 		end
+		telex.save_msg(msgid, msg)
 	end
 
 	-- remove old msgs from spool
@@ -256,8 +349,8 @@ function telex.process_spool()
 	S:set_string("telex_spool", telex.encode(spool))
 
 	-- return the actual messages
-	for _, msg in pairs(del) do
+	for _, msgid in pairs(del) do
-		telex.retour(msg)
+		telex.retour(msgid)
 	end
 
 	minetest.after(3600, telex.process_spool)

mods/terminal/init.lua

@@ -292,7 +292,7 @@ term.commands = {
 					from = c.name,
 					content = string.split(text, "\n")
 				}
-				telex.deliver(msg)
+				telex.send(msg)
 
 				return output .. "\n" .. "Mail sent to <" .. p .. ">."
 			elseif h == "read" then