Tested each sample.
Diego Nehab
parent
e394956cde
commit
52ac60af81
|
|
@ -0,0 +1,4 @@
|
|||
local CRLF = "\013\010"
|
||||
local input = source.chain(source.file(io.stdin), normalize(CRLF))
|
||||
local output = sink.file(io.stdout)
|
||||
pump.all(input, output)
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
function pump.step(src, snk)
|
||||
local chunk, src_err = src()
|
||||
local ret, snk_err = snk(chunk, src_err)
|
||||
if chunk and ret then return 1
|
||||
else return nil, src_err or snk_err end
|
||||
end
|
||||
|
||||
function pump.all(src, snk, step)
|
||||
step = step or pump.step
|
||||
while true do
|
||||
local ret, err = step(src, snk)
|
||||
if not ret then
|
||||
if err then return nil, err
|
||||
else return 1 end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
local input = source.chain(
|
||||
source.file(io.open("input.bin", "rb")),
|
||||
encode("base64"))
|
||||
local output = sink.chain(
|
||||
wrap(76),
|
||||
sink.file(io.open("output.b64", "w")))
|
||||
pump.all(input, output)
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
local smtp = require"socket.smtp"
|
||||
local mime = require"mime"
|
||||
local ltn12 = require"ltn12"
|
||||
|
||||
CRLF = "\013\010"
|
||||
|
||||
local message = smtp.message{
|
||||
headers = {
|
||||
from = "Sicrano <sicrano@example.com>",
|
||||
to = "Fulano <fulano@example.com>",
|
||||
subject = "A message with an attachment"},
|
||||
body = {
|
||||
preamble = "Hope you can see the attachment" .. CRLF,
|
||||
[1] = {
|
||||
body = "Here is our logo" .. CRLF},
|
||||
[2] = {
|
||||
headers = {
|
||||
["content-type"] = 'image/png; name="luasocket.png"',
|
||||
["content-disposition"] =
|
||||
'attachment; filename="luasocket.png"',
|
||||
["content-description"] = 'LuaSocket logo',
|
||||
["content-transfer-encoding"] = "BASE64"},
|
||||
body = ltn12.source.chain(
|
||||
ltn12.source.file(io.open("luasocket.png", "rb")),
|
||||
ltn12.filter.chain(
|
||||
mime.encode("base64"),
|
||||
mime.wrap()))}}}
|
||||
|
||||
assert(smtp.send{
|
||||
rcpt = "<diego@cs.princeton.edu>",
|
||||
from = "<diego@cs.princeton.edu>",
|
||||
server = "localhost",
|
||||
port = 2525,
|
||||
source = message})
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
function filter.cycle(lowlevel, context, extra)
|
||||
return function(chunk)
|
||||
local ret
|
||||
ret, context = lowlevel(context, chunk, extra)
|
||||
return ret
|
||||
end
|
||||
end
|
||||
|
||||
function normalize(marker)
|
||||
return filter.cycle(eol, 0, marker)
|
||||
end
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
local function chainpair(f1, f2)
|
||||
return function(chunk)
|
||||
local ret = f2(f1(chunk))
|
||||
if chunk then return ret
|
||||
else return (ret or "") .. (f2() or "") end
|
||||
end
|
||||
end
|
||||
|
||||
function filter.chain(...)
|
||||
local f = select(1, ...)
|
||||
for i = 2, select('#', ...) do
|
||||
f = chainpair(f, select(i, ...))
|
||||
end
|
||||
return f
|
||||
end
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
local qp = filter.chain(normalize(CRLF), encode("quoted-printable"),
|
||||
wrap("quoted-printable"))
|
||||
local input = source.chain(source.file(io.stdin), qp)
|
||||
local output = sink.file(io.stdout)
|
||||
pump.all(input, output)
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
function source.empty(err)
|
||||
return function()
|
||||
return nil, err
|
||||
end
|
||||
end
|
||||
|
||||
function source.file(handle, io_err)
|
||||
if handle then
|
||||
return function()
|
||||
local chunk = handle:read(20)
|
||||
if not chunk then handle:close() end
|
||||
return chunk
|
||||
end
|
||||
else return source.empty(io_err or "unable to open file") end
|
||||
end
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
function source.chain(src, f)
|
||||
return function()
|
||||
if not src then
|
||||
return nil
|
||||
end
|
||||
local chunk, err = src()
|
||||
if not chunk then
|
||||
src = nil
|
||||
return f(nil)
|
||||
else
|
||||
return f(chunk)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
function sink.table(t)
|
||||
t = t or {}
|
||||
local f = function(chunk, err)
|
||||
if chunk then table.insert(t, chunk) end
|
||||
return 1
|
||||
end
|
||||
return f, t
|
||||
end
|
||||
|
||||
local function null()
|
||||
return 1
|
||||
end
|
||||
|
||||
function sink.null()
|
||||
return null
|
||||
end
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
local input = source.file(io.stdin)
|
||||
local output, t = sink.table()
|
||||
output = sink.chain(normalize(CRLF), output)
|
||||
pump.all(input, output)
|
||||
io.write(table.concat(t))
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
for chunk in source.file(io.stdin) do
|
||||
io.write(chunk)
|
||||
end
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
#include "lua.h"
|
||||
#include "lauxlib.h"
|
||||
|
||||
#define CR '\xD'
|
||||
#define LF '\xA'
|
||||
#define CRLF "\xD\xA"
|
||||
|
||||
#define candidate(c) (c == CR || c == LF)
|
||||
static int pushchar(int c, int last, const char *marker,
|
||||
luaL_Buffer *buffer) {
|
||||
if (candidate(c)) {
|
||||
if (candidate(last)) {
|
||||
if (c == last)
|
||||
luaL_addstring(buffer, marker);
|
||||
return 0;
|
||||
} else {
|
||||
luaL_addstring(buffer, marker);
|
||||
return c;
|
||||
}
|
||||
} else {
|
||||
luaL_putchar(buffer, c);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static int eol(lua_State *L) {
|
||||
int context = luaL_checkint(L, 1);
|
||||
size_t isize = 0;
|
||||
const char *input = luaL_optlstring(L, 2, NULL, &isize);
|
||||
const char *last = input + isize;
|
||||
const char *marker = luaL_optstring(L, 3, CRLF);
|
||||
luaL_Buffer buffer;
|
||||
luaL_buffinit(L, &buffer);
|
||||
if (!input) {
|
||||
lua_pushnil(L);
|
||||
lua_pushnumber(L, 0);
|
||||
return 2;
|
||||
}
|
||||
while (input < last)
|
||||
context = pushchar(*input++, context, marker, &buffer);
|
||||
luaL_pushresult(&buffer);
|
||||
lua_pushnumber(L, context);
|
||||
return 2;
|
||||
}
|
||||
|
||||
static luaL_reg func[] = {
|
||||
{ "eol", eol },
|
||||
{ NULL, NULL }
|
||||
};
|
||||
|
||||
int luaopen_gem(lua_State *L) {
|
||||
luaL_openlib(L, "gem", func, 0);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,206 @@
|
|||
iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAIAAABMXPacAAAtU0lEQVR42u19eXRURdb4rarXa5LO
|
||||
RshKEshC2MLOBIjsCoMLGJhRPnUEcUGZEX7j4Iw6zqd+zjkzzowL6gzKMOoBRHAAPyQKUZQlxLAk
|
||||
EIEkQkhCyEoISegs3f1eVf3+qPTj0Z3udEJImN/Pe/rkdF6/V6/q3qp7b92tEOccfoT+A9zfHfj/
|
||||
HX4kQD/DjwToZ/iRAP0MPxKgn+FHAvQz/EiAfgapvzvQQ3DfviCE+rtTPYH/AAKouEYIcc4ForUX
|
||||
tXeKexhj6k8IIe2DvdUl0SYAcN7RGYQ63oAQ4hx8fBu6BXfC6vBcsHyDeNRi7cYboZQjBIRgl/lB
|
||||
KQcAQnyl+q1IAC9YU7/s2bOnsrKSUupwOHQ63cMPP2wymRhjGOOrV6/m5ORYLJbg4OABAwZYLBaD
|
||||
waBtQUsD34mqRT0hHc/abEpNjbWlxYEQCgw0RET463QEABjjjHFfyND/LEg737XsQpblhoaGioqK
|
||||
CxcunD9/fv78+ampqepgZFk2mUwBAQEYY6PRSAhRG7Tb7cXFxXa73W63W63Wn/zkJ4sXL1YfVHGB
|
||||
EFI5VZc0EDcwxjnnkoRbWhw7dxZt316Yn19TW9siyxQADAZddHRAWlrMffeNnDcvUa8nlDKEAGNv
|
||||
7ffbClCnoYoFFRFiIufn53/88cfBwcERERERERHjxo2LjIz0ZbaqFLXb7ZcuXZIkKSoqShAYY7xn
|
||||
z576+vpJkybFxcUZjUZfOJKKfQBACP75z/yXXtpfXX0JAAFIAAQAAXAADsAAZAA0dGjMa6/Nueee
|
||||
FEoZQsgLDfqTAFqWIstyRUVFXFycJEniJ6vV2tTUFBUVRQhxkb0q2TTS7xr9tNxG/bdjtAjl5eXl
|
||||
5ubW1dUhhJKTkzMyMkwmk0p4AMAYq91Tv1DKCMENDW0PPLBj797vEdJjrAfgjF2HP+d8B8YcAMry
|
||||
5VP//vf5Oh3h3OM66P8V0NTU9N133+Xl5SmKsnr16qCgIBc8MsbE5HXXgjqdU9oRie8YY5c2W1tb
|
||||
CwsLS0tLFy5cqEoILWnFI84rHGNUXW29/fYPCwsvSpI/pQLxntYNxxhjDIpinTNn1K5d/2Uy6Zwd
|
||||
cNWO+o4A7mjFGOfk5OzcuTMsLGzixInjxo2zWCwqIlSpAL2k47tMc+18FN8vXLgAAHFxce4Cqa1N
|
||||
njlzw9GjZZLkryiK6KP3twEgnY7I8tWf/WzCtm33McZVJVV7H3nppZf6BvXaL+rAFEVJSEhYvHjx
|
||||
4MGDDQaDykxAw1h6S38XLxUcRnRGnXyiM4cOHdqyZUtDQ0N0dLSfn5/4SUz/Z57Zs3PnCZ0uQFEU
|
||||
ANQV9jvIwxiTJOPp0xdCQgLS0gZRyjF2Hc5NXwEu866lpUWv1+v1enVBqFsnwWS0dLrZ4K7dlpSU
|
||||
ZGZmVlVVpaen33PPPYL1HzlSOXnyewCk+6gSo2OhocaCgl9GR1vEOtCO7qbbglQsY4yPHj366quv
|
||||
nj59GjScWtBGq0f2mVHBZbVxzhMSElatWvXzn//cORUAANau/Y5zB8YYoLsUQJxzQqSGhqb1648D
|
||||
gFClXO+4eSNUZ9alS5e2b99eXl4+d+7cqVOnCrl361hvOt2LCNWlttY6bNjbTU22Hk9WhBDnjhEj
|
||||
IgoKVoqdc1+vAFmW//WvfymK8uyzz86aNUvlP72HPrjBWaR2RkgIoXeJ2ZqbW9nUdBVj0uPGOecA
|
||||
ujNn6s+cuQRui6CXd8JaJUedSsJUEBoaqtfrtdd9p4HQ3rTGL9UE1ik2BZ/trmnMRePinAFAQUEt
|
||||
AMMYuXMP34EQRKnjzJlLqakRLr3uTQJoJarLzigyMpIxJiStVr/0pTXOQdgAMEaEYACOEPb+tKCU
|
||||
UOEVhYq9qKCKTwYyzW0XL169cUaNEAJglZVXwc2Q3msE0GKfEFJYWGg2m+Pj41UtyMeJr8W7olCB
|
||||
dFVS2mxKZeXVqqqrFRXN9fVtDQ1tbW2yw0EBQK8nJpNuwABTWJjfoEGB0dEBMTEWk0mHEBYPU8oY
|
||||
Y04S+roEbTalt1Bkt1P3i728AjjnhJCjR49u3rw5IyNDEACcvBW8ajgqRhSFCUsvQhghVF/fmptb
|
||||
efjwxWPHqs6da6iutlLqAFA86yQIQCJEHxkZkJQUMnFi9JQpg9LSYsLD/THusCtw3mHR7JIMfn66
|
||||
3sKP2dxJU70sAzDGBw4c2Llz5/333z958mRVqfD+lBb1GCNhxa2oaP788x8++6z4yJFKq9UKQAGI
|
||||
+CCkw1jvqVkhPylllZVXKivrv/22EID4+wdMmhS9YEHKggVD4+KCxAqjlHkig9DfASA+PkismO7r
|
||||
oNeAMQ6A4+ODwG0K9o4aqtoajx07tnnz5mXLlo0ePVplO12iXhjZMUYYI1mme/aUrF+f/9VXJTZb
|
||||
CwAG0GFMhDHLxfjlHQTTF/KTMQogAzCDwW/27ITHHhs/f36SXk+8GO4VhUkSzsoqmTv3XxgbbkQI
|
||||
A3BJQmfO/DI5eYAQhL1JAK0l68qVK1euXElMTOyS6av6EqViI4bb2+WNGwveeCO3uLgSAAAMhBCA
|
||||
Dh/TjQMhCABRSgHsAJCUFL16ddrDD4/289OrfQDNahBGiKYm2/Dha2tqrAj1YCcMAIAxYsw+aVLs
|
||||
kSMr3G2IN7QPcOqFXJ3IISEhCQkJvmBfaIeKIqQifPDBiREj3n3iiW3FxTUYmwgxCWT1FvYBgFJO
|
||||
KQVAhJgwNp07V7ty5afDh7+7fn0e50AIVhTGmNZiCIrCgoKMixYNB7D3aCcMTvalPPjgGNEHl597
|
||||
vgI8Gd8FL/JkLnaf+IcPV6xatScv7zxCEsYGdQd0k6HDvs2Yg3PH6NFD3npr3vTp8Wqv1D0Hxqik
|
||||
5MrYse+0tFCn48X3LSTHGDMmJySEnDjxy4AAfa+tAK1yWVpampubqxJDMLhOub9W2BKC29uVX/7y
|
||||
i/T09/LyygjxQ0hPKe0T7AMAYoxTShGSCPEvKKiYMWP9E0/sbm11iKXgHAIoCktMDHnxxVkAbTpd
|
||||
t9DFnahW/vSneQEBHYzOBS09IYA62THGra2tmzZtOnfunO9PCeF25Ejl+PHr3n13PyE6jI1O1Pex
|
||||
dQgxBpRSjA2E6N9//+DYseuysyskCVPKBTsiBDHGn302ffHiCbJs1ekkJ3K7GC5CSKfDlFrXrJm1
|
||||
ePFwShnGnYyuJwTQ+vk2bdrk5+e3ZMkS9Scv2GeMU8p1OvLOO0enTn3v7Nk6QvwpFQbRfjTMIcYY
|
||||
pZwQ/9LS+mnT3n/99e8kCQtmKNYB53zTpkV33jlGlpslSWzIPZFBhKUQjLksW596auZrr92hYt8d
|
||||
Pz1cAQKhmZmZpaWlS5culSRJsKNOJYrWqY0xeuKJz3/1q38DYIz1lIrNYT9gHyFXAxGlFGM9xtIz
|
||||
z+xctuwzYUESXnXOQacj//u/S3796zsUxU6pDSGQJEKIsHB0fAhBkkQQ4pS2Ygyvv77o3XfvFNjv
|
||||
zagIVZLs27cvMDBwwoQJqpHHE98Xno3WVvlnP9v65ZcFkhSgKKybAu0GgQMgse2iVIQviIFjjDHG
|
||||
YnvccZskYUWxzp49cseO+y0Wg+i82DFIEj58uOL55/cdPFgKYHfuDcUoGAAFYISY77572B//OGv4
|
||||
8DBFYd6jg3pIAE8hCF6w39xsu+uuTdnZZyXJv2+x34F6xhjndgAOoPfzM5nNEqXcarXLsg1AAdBh
|
||||
rIcOB5GgQcukSQlffPGL0FCTGIJgSmI65+VV79xZnJNzsby8UQ3MSkgImT49PiNjWHJyqBrC5d3u
|
||||
1A0CuHstvOv7KufBGFmtjnnzPsrJKZEkP0WhfTnxnV1t0+mMs2YlLVyYMnFiVHS0xWzWUcqammzn
|
||||
zl359tuyf/+7sKSkGiEJIT1jFAAkiShK68SJg7OylgYFGcVAAECrqiKEZJm2tysIgdmsc14EWRY2
|
||||
FY/q+A0RQG3Re2yIerMsszvv3Pj114WS5N/n2McACufKz38+/uWXZ6SkDHDvs4rH7duLXnjh69LS
|
||||
GkLMlHIALmgwbVry3r0PGwwd4T3gNDcJkqiUUC8SgjEWPoyuba6+CmFtAMH+/ftra2s7COjVuim0
|
||||
iEcf/axfsI8x5twRGGjYufPhrVsXJyeHUsrEdlf7oZTLMiUE33//yFOnVj7yyBRK2wgBAKQoVJL8
|
||||
Dh78YenSHerqV13cOl2HhUr1DmGMdDpSX3/p22/3C1+3FnU3RAC1obNnz+7atau9vd1L007WzwnB
|
||||
r756YOPGI/0y9xmTo6IsBw8+vnBhiixT4dIRWNN+CEE6HRF7LoOBbNiw4JVX5lNqwxg5aeC/deux
|
||||
F1/cRwimVJV/AM79ppAK6opvb2/ftWtXSUlJl9iHbsUFiXds2rQpOTl52rRpnoydzoAfJkk4M/Ps
|
||||
Y4/twNjotIH0ndQFYP7+ur17l40ZEyHLVJKwpy26+q/Q7hWFzZw5uKVFyck5R4gwjQDGhgMHzqam
|
||||
Ro8YMVBs472YuYKDg69cuVJQUJCWlubi5nQHn1aAuu5OnDhRU1MzZ84c7/cLda2mpuWJJz4DQJx3
|
||||
14Ryo4AxAnC8+ead48dHORxUhIx7R4Rzb48IwYyx116bm56eRGm7sMFxDgDSU0/9b0VFsyRhL/YS
|
||||
8Yrbb7+9trY2Ly9Pxd4NEUCFc+fOTZgwYeDAgWL6u9+g2kcB4Omnd1dVNRCi57wvN7rC/mWbNWvo
|
||||
8uXjKWU6He5SErrQQAjb116bCyAJAwnnjBBdXV3jr36122WY7sAYCwsLGz9+vOBCXbzURy3Iydap
|
||||
oijafIfr7+kw4UoS3rLl1H/912ZCTJT2tZkBIcS5PTNz6fz5yaIzvicMqWillEsSzsjYsnNnASEm
|
||||
oRQRgilt+/DD+x9+eKyzZe6GhA7M2O12Qoga7O3pdb6yIPEXY+w1qodzziUJNzXZXnghC0ByKgJ9
|
||||
BxgD546UlIjbb08AAEKuCUwfQTu0hx4aDYDUKcoYB9D9/vdfX77c5oURiZWk1+tFYD14FcVdEECr
|
||||
fbq8wH36g9Ph8Ne/ZpeV1fU581HRp8ycOVinI6pVuQftCH1/6tTYoCALY1SIUs45IfrKyvo///mQ
|
||||
kx6uyHVHTqc49JUA2na1Ar2zUXHOQZJweXnTO+/kAhj7nvmoMG5c9I08rlpABw70T0oKBVCc4xV+
|
||||
JNM//nHk3LkGwdw6fVz7txc2YoyxrVu3lpaWImecs4fbOACsXftdc7OVEOlGwgh6DJwDAImNDdTi
|
||||
omcghhMTYwFg2glNCGltbX3jjRzoLNhWizSEUHl5+datW51G307AGwFU/amqqur48eOSJHm9EyQJ
|
||||
V1Vd/fDDEwCG/jLxc84BkNEoAXRD8HpoCgDAZNJdP5PEIjBs2lRQXt4kFoEXFi9J0vHjxysrK8GD
|
||||
PurTCvj+++9jYmJiY2O9CHQxFz766ERjYxMh0s1OO/AEIoDH4VBUDN4g2GyK20zihEhW69UPPsgD
|
||||
z4tACIOYmJiYmBgRkd8pdEEAsXssKioaOnQoeBAj4pokYYeDbtpUAKDrD+eiOmwAoCIKE3ywBHgd
|
||||
OwKAqqqrAC68XvBh/ebN37e3y5KEPWOGA0BycnJRURFowgOve0uX/bBarYqiCAJ4gI44hm++KS0q
|
||||
qkVI31/TX2AHAPLza26kCTU5oKGhraTkCgBxGRHngLHu/PlLWVkl0FmwiRaGDx8uy3JTU1Onv3at
|
||||
hgYEBKxevTo2NhY8y3TRvU8/PQ1ARZbnTcaytw4DSPv3lzHGvMxN39qB3NyLDQ3NGEvubYjYrU8/
|
||||
PeOpBRVXMTExq1evDgwM7PQ2bwRQce2Siu4OkoStVntW1vn+5T8AwBhHSHfqVPWBAxfAq5biCdSg
|
||||
MQDYvPl7pwrE3V8EoP/669LGxnZP+qgAQojJZPLkG/BIAHXiMK/bWTWO6tixqsrKKwjp+rv2hBgk
|
||||
FWqi6Ex3nU6UMknCBQW1//73GQADpZ1MKc4BY6murik3txKgI4PBS8ue3ANdywDkBPDo/AIA2Lev
|
||||
FEDpNPSlbwExxhEyff756W3bTksSVhSP4RpuA7mWmgAAzz2XJcs2LxGJgtL79p33gjoXBLpDFwRo
|
||||
bGwsLi7W1gXopAmMACAn56K7sOonEGUbpJUrPz93rkGnI7JMVX+Wx2ec2JdlJkn4j3888OWXZwgx
|
||||
ednQcM4ByHffXVSR4OEeYIz98MMPjY2N3SCAQHphYeG2bdu8+h0BY9TY2H7mzCUA7+o/BwBJwuKD
|
||||
8Q1F3HsFYVWWLl+23nXXxoqKZkED1UnrptJ0/KsojFKu15O///3Y73+/F2NTp8zn+gelwsLLly61
|
||||
CiO2xw4htHXr1sLCQnBj6dhz0wAADQ0N4eHhXpawuF5aeqW+vsVrKnOHl0pRWsSHMYcz1vWm0IAx
|
||||
hrHh7NlLU6a8n51dIXwyAsXOND+uutFlmQonEsbouee+XrlyB8Z6sey9vINzQAg3NbWWlDQAeHMP
|
||||
IIQiIyMvXboE18cVgpcMGTHrm5qagoKCwHMqj2iqqOgygEyI5FkjRgA0JMT/oYemMMbNZik7u+Lw
|
||||
4dKbKbQ7aFBV1Txjxvqnnpry/PO3RUT4u3gyEOpYxAihb74pW7MmKz+/lBATpeCLFw9jRKlcVHR5
|
||||
ypRY7wMJCQnpdCvQBQFqampGjRrllQAcAM6fvwLAvOTxYIwYYxER/m++OU+WqU5H/vzn7MOHfyDE
|
||||
IIzGN48GCOk452+/vf/DD/MXLhy+cGHK2LER4eH+BgNhjLe0OMrKmg4evLBly+mjR0sBgBA/Sn2N
|
||||
GxNDPn/+CnheAeK62WwWDjIXNHZBgGnTpkVFRUFX4ebl5U2+ONc45yIwRKcjvZh54R1FnDPOESF+
|
||||
Vqt948bcjRuP6HTmsDA/k0lijDc12RsbW0SQIcZGABBJHD5uZYTtr7y8CTy4SVS8DR8+XPASn1iQ
|
||||
2sqUKVPUnAsPdwIA1Na2+DhfCMGS1FHWrk8IAKJjlFIATIiZc5BlWl3d6JzjCIBIkr8QBt0NHhDR
|
||||
QLW1LeDZ9C2iZuPi4uLj413Q65EAmjypTqrruOAUABobbW4Wq1sN1KhCBIAQujZwkSmlva27LTc2
|
||||
2gDAwxS9LoPapRwXdOkPgK58GkL/bWlx9GuAfzeQ5RyaWu/gWnC5Om7fmxMsqLXVIaLYfbv/OvDG
|
||||
grR830vrjHFZ7gPvu8hX6ZhBIkyhM6q73MY830Mo5ZxTkQ/sXBmYENJVRTJXbMkyY4x7spZ5R6a3
|
||||
fUBLS8uWLVvq6+vBqzlFNQfdzG2wCM6hYg9BaZsT+7yz2xTnbe2aeobqDYKjUkVp4dxuNOojI4Ni
|
||||
YkIiIgJNJj3nsqK0cE67lRPp3RAkfrpy5cqWLVuam5tdEOUtU16W5ZMnT6alpYWFhXnxhWGMhOHX
|
||||
R5NLDwAhxLmSmDhw6dIxisIaG9vffvuou5EAIcS5nJoac999IxWFVVdffe+945p7OIDI226LjBzw
|
||||
4INjfvrTxKSk0MBAA8ZI5AqUlFzZu/f8Bx/k1dZewdjkm2OVq3GPngiAEGptbT1x4oQIKtQi0xsB
|
||||
JEkym83ecSra0uvJTfUBYIwoVZKSQl54YRoAlJc3/f3vx9yttOK21NTw55+/DQAKCmrfe++YBoMI
|
||||
IWDM9sQT6X/961x/f9cAJ4vFEBUVMH16/G9/O3X58s+2by/A2OidBsJwrdcTX5Q6s9ks/Oq+pqmK
|
||||
ux0Oh1cCdHS9D5wwKsZFioTnLl2z7WgvY4w4t/2f/zNt3bq7jUZJWEnb2uTy8qZz5xpqaqxCkDoc
|
||||
1GIxbNt23223JTDmS342t1gMahKcJ7DZbACg07nW6/C2AvR6vUhE7Wq0KDTUBNC9ALQegLrKnUmK
|
||||
ncO11S1h7UXG5Li4ga+8MotzTgi6etX+4ovf7thRePlyG6XcYCCDBwc//fRPHntsvMNB9Xry7LO3
|
||||
HTpUKp72/C4AYCEhRuiq8Ep7eztCSK/Xd4MAGOPHHntM1PL0nH8KABAdbfEgFW8VEEabO+5I9Pc3
|
||||
tLXJZrPu/vs/3bPnBMZ+jImodKWwsOrxxz9ubZVXr04DgPHjowIC/K1WG0Letzg8OtqiosIdBOqS
|
||||
kpIef/xx99CeLvwB4eHhQgx42oWJ9e6s6dLfaO4KxoyJBACzWXfgQNmePWckKciZ44gAMCEGAOMn
|
||||
n5wUN1ssBn9/PYA3didsQaIOjXcsmUymiIgI9xsk762L8nVqRpj78+JKSkooAOmrKgM9AcY6nPWt
|
||||
rQ4AyM4uBxD7gA59X5hFAXBbm+K7QUIUAkpJGQDXMwltipxKg04R6G0jxjVlNzyB2AkPHTqAEEM/
|
||||
BoN2CZxzAN2nn5749NPjAICQjhADAEeoo2QQ54xzBaAlPn6okyRdj4UxBmBwEuAa6kGjC6hGuk43
|
||||
Yt6iDcUKsFqtfn5+nuISRVNxcUFxcUGlpZcRkm5VixAC4BgbCUGEYIdDobTdyV4wgC4gwBgVFTB9
|
||||
+k9efHG6ry0i4JzGxAQPHhwMzrmoTSRV+YdLQrX2YhcEqK+vX7du3YoVK8LDwz3xOEqZwSCNHRtR
|
||||
WlqLsa6v7Mw9Ac5BURRZtpnNAWPHJo0eHT506IDBg4NiYizh4f4DBpj1euKJV7iD2HaMGRMhSj6p
|
||||
GawIIVGhua2tbefOnQ888IBIquCaepLqsujCHxAYGMg5r62tDQ8PBw9iQEz5GTPit28/0d8Y9oZ8
|
||||
hDDnsr+/Yc2a2cuXj42OtrjkPAuk1NW1DhhgliRfeCkC4NOnx6tI4M6ikQcPHszOzo6MjLRarcXF
|
||||
xXFxcRaLRSS3MsbKysqioqLE8RHehDDn3Gg0hoWFlZeXjx492jOlOABMnz4Yof7MCegCVQhxLkdF
|
||||
WfbsWTpq1EBhvUEItbfLVVXWysqrZWWNZ8827N9fTik7cuQx8MG0RSkD0M+cORg6WLHgchgApkyZ
|
||||
Eh8fn5WVxRj7/PPPbTabxWKJiopKTEwMDAz8+OOPn3zySXEgiDcCCGIOGjSouLgYPAgl9YyUUaPC
|
||||
x42LyMu7eMP17W4UtPsvFUSm0IYN944aNdBmU4xG6fDhin/841hOzsXKyquybAdQMAbG6MiR8T7y
|
||||
H8Yco0ZFjh0bKf510gA45xaLJSgoqLq6OiIiYuTIkefPn7948eKFCxf279/f0NCQkpISGRkJLn6J
|
||||
zpArVMyU9vZ2tR5Kp3dSyiUJ3XNPSl5eGUJGgJu7DrwkmwLwyEg/l6uEIErtkycPmTcvyeGgRqP0
|
||||
t79995vf7EKIca5T62ASgh0Ouyj02hWIIgjyXXcNxRiJkihOSndwaUrpnDlzBMYSEhKGDBkixHJj
|
||||
Y6PZbAY199UL9gVPTEpKSk5O9u6cEZczMob/z/8cuHkZ8S6ntbj/DsABsJiSLmMBoGlpMQCg15Pq
|
||||
auvLL2cBSJKkUxQm3DLCNwDABUftCkSahnHx4hHunXGWLcCHDh3Ky8tDCA0aNGjq1KkiwCc0NFSV
|
||||
85zzLjxiWsekp4Q/5KzNOXJk+OzZgwEcvgQoIoQAsBqn5eXj3CJdA6NRMplc3B8dWbQDBwbOnDmk
|
||||
09GEh/uLb+XlV6xWGWNJRGupN0gSAXAMGxaqGbtHCzyAfcaM+HHjotQCNi5427VrV2ZmZnJycmJi
|
||||
Yk5OjsPhOHnypOpcUbUgn6xa2mM/PBn9Bd9/9NEJaje8E4BzGaBFUVrVUC1PH84V56JmAKAoLDzc
|
||||
f9y4CACbXt9R+EGSCCEYoPU3v7ltwACzqCbtAlZrh1k3IiJAr8ecc0lSH0eSRByOlvDw0Fdeud05
|
||||
duHkwm7hNuI7f/TR8eAWgC12r3V1dceOHVuxYsX8+fMTEhLi4uIGDRqUm5u7bds2uD5+ouvSxej6
|
||||
2kyeQDDBBQuGjRoVfepUDcZ6T6JYrI/x4wc98sjtAQEGr1l/YDJJu3efLS6uA5AqKpplmYrH//Sn
|
||||
eXPm1FitzSK0i3PKOaxcOXvNmnS1sI8WKQD4++9rAcDhoEOGhDz2WNq77+5jTM8YEtoj5zBpUuLG
|
||||
jfeKoiqEYEIwxgqAnXOjtmAlQpgxx9ChkYsWjQC38A6BpbKystDQUBEGcezYsYSEBAC4995733nn
|
||||
ncrKypiYGLXUQBcEUGNSDh482NzcfPfdd3dapAA5yyHqdHjVqsmPProNIYO7KBaF6MUsnjVryJw5
|
||||
CV62PMLxK0m4vr61uPiiJPn98EPd4cMVM2YMttuVSZOi8/OfWrs2Ny+vRlFYQkLwL34xZt68RADY
|
||||
uLHgrruSQ0PN6pZQxPLv23e+pKQhMTFUlunatT/9yU+it207U1fXoteThISQBQtS7rwzyWCQGhvb
|
||||
jUbJaEQGg/SrX6W/8UZua6ujudnmHAvHGFOqPP30ZOFUEDWxtKgAAD8/v6tXrzocDs55ZWXlrFmz
|
||||
AMBisRiNRhf/iq95wnq9/rvvvrNarWpghadF8NBDY1JTB1HaiStD6KyEYEKQpyqCngBjBMDWrNnb
|
||||
0uIwGCRZpoMHB61dOz87+5EjRx7bvHnRHXckAMBf/5rzyiv7goONoIlY5hwwJm1tbatXfymyORnj
|
||||
Dz00eteuJUeOPJ6dvfzDDxcuXJhiMEj5+TXp6RvKyhoRQna7smpVWlXVMw8/PAbARggSyg+l9pSU
|
||||
qGXLxrlMf62eMmzYMKPRuHXr1ry8vIEDB0ZHRwPA6dOnKaXiu08uSe1948aNy8rKOnny5G233ebJ
|
||||
LCoWgV5P/vCHmYsXb3KZzgCorU0+dOiC78YixlhgoLG2tgUAKwrD2HD8+IVZsz745z8XpqaGq3HO
|
||||
oj/Nze0vv3zgzTe/iY+PPHSowmzWnTp1SdsUxsbMzNNz5360bt09Q4YEO+cQF1HTly+3vv320ddf
|
||||
P9Ta2rxhw4m//W2uWoxAOFydwQ3AOX3xxZkmk+v0V3l1W1ub2Wx+5JFHNm7cKPhPTk5OTU1Nbm5u
|
||||
RkaGwWDQchGf4gkFF9q9e/fJkyefe+457dmCbljukEJ33bUxM/MMIWZnpJ/qrunBeQgEAKsBDYzZ
|
||||
JUmaPj1xxoy4uLggnY5cvtyan1+7e/cP9fUNGJsZY863IAA1XxyphVSMRuOcOUnp6bExMRaEUG2t
|
||||
9ejR6qysksbGKxibADBjjgULRt5zT4rJJFVVWf/1r/yiolqEJIQQY+1z5qR89dVS7cFsKkIF9r/4
|
||||
4osFCxbodDpZlk+dOnXq1Kn6+nqz2Zyeni7OI9VObp8IIO6ur6/funXrkiVLhCbrKVZXBBsXFdVP
|
||||
nPiP1lbFibsOGvTAaaNWkxT/OQ9BsQOoQZxC2OjV8Gz1LW7hPeJxUT6ROTmw+rhOhOUihDi3qSH1
|
||||
AHonq+BGI8rNXTF6dIRaDVQb+EYIaWxsfOutt1asWBEREUEpdT8IE67Hgk8pSuJLaGjok08+6QX7
|
||||
HS1ipChs2LCwV16ZA2BzMQyIXU+3Pi7dYYxxDoQYJcmfEDMhJvEFIaI66zXPos4eR86nTNc/TtXH
|
||||
CTGpjSMkidgTgPY//GHW6NERatF3AfX19QL7lNLg4OCwsDCRGAwaxb2trU1dKNpJ373kKRfC+MaI
|
||||
ThPi52RE/6HACSGUtt1+e0pW1jXmI5Bgs9lef/31gICABQsWDBo0CCH0zTffHDlyJCUlpbGx0Waz
|
||||
ORyOpqamMWPGLFy40L3OW/fKVoLGeOuJBiLaUj2BdPLkdRUVTRjr+7tAdM+xL0rQR0YG5OauiI0N
|
||||
FEPT8pPa2tq9e/eeOXNm6NChCxcuBIC33norPDw8MDDQZDL5+fkZDIbU1NROmUf3YtmcWZy0tbU1
|
||||
ICDAMw2u1e07ePDCnDkbZFn1Cv1n0aDj9BiEWFbWstmzh7gXylLnYmVl5e7du0tLS8ePH19RUbFg
|
||||
wYLk5GRtbdtO0dW9mnGilYKCgrffflsEunRKvw5nm4QVhU2bFrdhwyIAu6hZeMvGrXQ6XBHKyLn9
|
||||
/ffvnT17iKi+6C5UBURHR69YsWLFihX19fXV1dUHDhxoaWkRKoOQLp1O1m4fZ4sQCgkJOXz4cFNT
|
||||
0/Dhw9UW3TNDOOeEYEWhY8dG+vub9+49TYj+epXmVgYOgCQJUdr6xz/euWpVmkjs6TQHpr29/bPP
|
||||
PtuxY0dRUdGkSZOmTZsWExNTVFSUlZVlt9tjY2NFPFanWUbdI4DQeXU6XVhY2K5duxISEgRf8xCa
|
||||
isQ5RpTy9PRYQvTffHNGkv4jaNCBfUVpfeGFef/93zO0ey4XwwNj7P33329sbExPT9fr9YmJiQI/
|
||||
aWlpFoslPz9/xIgRJpMJPOjg3ZYB4NRwPv7448rKymeffRa8pvAh5ylVkoT/9Kfs5577nBAjY7jv
|
||||
y8n5PkqEMMac0vaXX57/hz9M91SCXjipjh07lpWVtWbNGrWcoSzLe/bsSU9PDw4OppS6HMbuAt07
|
||||
yE3b0J133nnlirfsQO39hICi0N/9Lj0kxLRixQ7OMSG6W1I35RgTzmVK6TvvLF65cqIn7KuGkKqq
|
||||
qoiICL1eL8syxlhUNTlx4oSiKPfee2+X7+v5ESYWi2Xw4MEuEqlTd42TBliW6eOPj//yy0eCg42U
|
||||
tkuScKrcImJZJPITxtoDAgyff7505cqJskxdsK8OkznPlIuKiqqoqGhtbRWRz4qi6HS66dOni6TU
|
||||
Ls9w7DYBtL1Rjy1xiezw9IgkYVmmc+cmHD/+1MSJgxWlhRDo6flcvYx9jDEhoCgtY8bEHj/+5F13
|
||||
JQudx9MACSFiso8dO9ZsNn/44YeiUqu48/Lly2qCu/cXd1sLguvLMoovly5dUhTFZDJ5OstE02+s
|
||||
KCwkxLRs2Vi7nWRnn+dcIUTv9Oj2PUfqyBdjzME5Xb165iefLB440F/oPNrxav2INpvt8OHDR48e
|
||||
tVqt0dHRI0eOzM7OPnjwoF6vlyQpNzf38OHD9913X1BQkJcM347GbySpSDWUbtiwwWq1Pv300ypt
|
||||
vItlcWCLOI9lxYrdp0+XI2TEWHKu674hA3dGSimc21JSYtetu+v6s9w6hgiaEAWEUGNj4/r16yml
|
||||
AwcOLCsrE5bnkJCQL7/8sqCgQJZlPz+/u+++e8SIEVor6U0hgIrQq1evvvHGG3FxcUuXLgXPSpH2
|
||||
EVU1stuVd9459uqr+5uaGvuKDNeh3mIJfP756atWpQkPl/ASg5PBqtNfDeh8//33CSHLly8HgLa2
|
||||
to8//ri0tHTVqlXh4eF2u729vT0gIEA1gnYZ5dgTFnQdARFijBmNxmHDhu3Zs+fixYujR4/2/mIt
|
||||
OxJG3alTY5ctGwugP3WqzmazAiCMJe8FYHqGdwDkFKoK5+1+fuaVK9O3bFk8b16SKJWrMn2xshlj
|
||||
Fy9erK2t9fPz0+v1CKGmpqY9e/YsWrQoKCiIUmowGMaNG1dcXHzmzJlJkyYRQoxGI3Kecuc9lkfA
|
||||
jZ4nrHY0PDx8+fLl3377rcPhMBgM4HUdqNNKnISgKCwszO8vf7n9179Oe++9vPXr86qr6wEAQC8E
|
||||
XbdOse3sdcI9KU4HdQBARMSARx8dt2LFhOhoC2PcRdcUgyopKdm5c6fVahWCbfHixampqeJXNW1L
|
||||
WPx/+tOfrl+/vq6uLjw8XCj+XmoL9DIBtNSOj49ftmyZOgzBSbyXOVBrjgosRET4v/TSjDVrpmRm
|
||||
nvvoo5P795e1tVkBAEAHIKk4UvPcPaFbcA6V0XGuUKoAcJMpYNq05IcfHn333UNFlqTgOcLCIxoU
|
||||
6M7Pz//kk09mzJiRnp5OCMnMzBTFZgIDA+Pi4r766qvhw4cTQhRFAYCgoCBCiN1uB429wUffU68d
|
||||
6KyuXK28cr/i4XEQfFk9XlkMoLraundvyZ49JTk5FysrmwDEKWDCQyk+1zXpNHIw50ds9PRRUUFT
|
||||
pgyaNy9x7tzEmJiOoGj1CGn3GOnGxsa//OUv99xzT1pamjYmU8yn+vr6N998MyEh4cEHH9TpdAih
|
||||
L7/88uTJk7/97W99n/i9TACVDNfaRSgvLy8iIiI6OrrL7bg7ISnlCF07q6u9Xf7hh4a8vOrvv68r
|
||||
LKyvrLx66VKr1eqQZVlzJh4CwDqd5O+vHzjQLybGMmxYWGpq+PjxUcOGDTCZdFoFzNP5aoKlZGdn
|
||||
Hzhw4He/+506lxFCLS0ttbW1JpMpOjq6srLygw8+UBRlxIgRjY2NFy9efOSRR4YMGeLLIeIu0Jtn
|
||||
yrsYab///vtt27YtW7YsJSVFXQq+tAAA6lmaooSM0SiNGRMxenQ4dIh93txsa262NzfbbDZFVKrQ
|
||||
6bDRKAUGGi0WQ1CQ0WVqi7P7xKmFWut8px0wGAytra1NTU2hoaGKopSXlx85cqS4uNhms1FKp0yZ
|
||||
snjx4meeeSY3N/f8+fMhISH33nvvwIEDuQ8ZXZ0MuRdXgArq8L744ouvvvrqjjvumD17ttejNzy1
|
||||
I8JAROHBDtYv+IYXh6jTRX7tLFRN8lAXJdWdC679jTfeYIwlJiaWl5c3NDRERUVNmDBhyJAhZWVl
|
||||
27dv/8UvfjF27NgunS39QwAt98cYnzlzZvPmzUuWLBk1apSWn/asu2pvPVVkVaN3tP92t32EUHV1
|
||||
dWZmZnNzc0JCwsSJE0U0lfhp3bp1gYGBS5YsURRF3eX2gPvfLAK406ClpcVgMOh0Og361KolXWvK
|
||||
fQlaa4/LF+HVkiTp7bffjo6OzsjIELLtBvvfwyPNvYM6u4Uyqk2yFIYUdffgyX7Xl6BqONq9K3cm
|
||||
1MmyzJ1nF0qSdOjQocrKysmTJ4NTON/g7OlNIawFtVtaHU5c+eijjzDGGRkZAwYM8FE43yTQmnVB
|
||||
M+XVBVpXV/fBBx/Mnj07NTX16tWr+/bty8vLe+CBByIjIz2dpNZtRPXZ7FOXc2lp6RdffFFRUTF2
|
||||
7NhZs2aJBNjr+tQj8dDdzqjTXFWRtdtGZ2CHsmvXrtzcXJPJpChKWFhYRkZGbGyslwOsuwt9vfxV
|
||||
Mpw9e3bHjh1JSUmLFi1y2eyoJtxep4SLyFH/LS8vz8zMHD16dHp6urtuc+nSpbq6uuDg4KioKME5
|
||||
u9xa3ooEUMejVmJUFEVRFJEuK8Zjs9lUY1ZH/9yQ1bP3goa0Ku7sdntOTk5+fn59fX1CQsIdd9wR
|
||||
FxenfbX7svDdyuYj3CwZ4A7qNk0MQARTqmfNAYDNZlu7dq3FYpkwYUJSUpI4ckKrh2hnnIvBw9O7
|
||||
tPeD2ykuIm8rMTHxoYceEjsp7SMuEkIVxb27KPtHA3HX9gTDPXv27MmTJ8+fP2+1WtPS0jIyMnqw
|
||||
uXdRIgU0NzdXVlYWFhaOHz8+ISFBZXoqu+uyQupNgr5bAVpwd2oCgCRJw4YNGz58uKIo586dcxED
|
||||
R44cqampGTRoUGBgoMViCQ4OFhsLLaIZY4qiUEpFjSN1J7hjxw5ZlgkhgYGBqampLj1RVaA+EP6d
|
||||
oKJ/dXABWg4LTkah5d0iSe3YsWMOh8Nms8myvHLlyujoaDGR29vb169f39LSIqwI4eHhK1euBKdh
|
||||
ubq6uqioaMiQIZGRkULegJvZqh93grcEAQRop7N2q6xlVoyx1tZWq9U6YMAAbSDU8ePHEULiANOg
|
||||
oKDY2FithHCRFv0y0z3BLUQAT6C6d7TaIfiAR5c9bZcBA/0C/wEEEKDtZ6duHy1a3Wtk37LwH0OA
|
||||
/1fhphjjfgTf4f8C4VLHz/5KLxoAAAA8dEVYdGNvbW1lbnQAIEltYWdlIGdlbmVyYXRlZCBieSBH
|
||||
TlUgR2hvc3RzY3JpcHQgKGRldmljZT1wbm1yYXcpCvqLFvMAAAAASUVORK5CYII=
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
351
gem/ltn012.tex
351
gem/ltn012.tex
|
|
@ -6,7 +6,10 @@
|
|||
\DefineVerbatimEnvironment{mime}{Verbatim}{fontsize=\small,commandchars=\$\#\%}
|
||||
\newcommand{\stick}[1]{\vbox{\setlength{\parskip}{0pt}#1}}
|
||||
\newcommand{\bl}{\ensuremath{\mathtt{\backslash}}}
|
||||
|
||||
\newcommand{\CR}{\texttt{CR}}
|
||||
\newcommand{\LF}{\texttt{LF}}
|
||||
\newcommand{\CRLF}{\texttt{CR~LF}}
|
||||
\newcommand{\nil}{\texttt{nil}}
|
||||
|
||||
\title{Filters, sources, sinks, and pumps\\
|
||||
{\large or Functional programming for the rest of us}}
|
||||
|
|
@ -17,30 +20,31 @@
|
|||
\maketitle
|
||||
|
||||
\begin{abstract}
|
||||
Certain data processing operations can be implemented in the
|
||||
form of filters. A filter is a function that can process data
|
||||
received in consecutive function calls, returning partial
|
||||
results after each invocation. Examples of operations that can be
|
||||
implemented as filters include the end-of-line normalization
|
||||
for text, Base64 and Quoted-Printable transfer content
|
||||
encodings, the breaking of text into lines, SMTP dot-stuffing,
|
||||
and there are many others. Filters become even
|
||||
more powerful when we allow them to be chained together to
|
||||
create composite filters. In this context, filters can be seen
|
||||
as the middle links in a chain of data transformations. Sources an sinks
|
||||
are the corresponding end points of these chains. A source
|
||||
is a function that produces data, chunk by chunk, and a sink
|
||||
is a function that takes data, chunk by chunk. In this
|
||||
article, we describe the design of an elegant interface for filters,
|
||||
sources, sinks, and chaining, and illustrate each step
|
||||
with concrete examples.
|
||||
Certain data processing operations can be implemented in the
|
||||
form of filters. A filter is a function that can process
|
||||
data received in consecutive invocations, returning partial
|
||||
results each time it is called. Examples of operations that
|
||||
can be implemented as filters include the end-of-line
|
||||
normalization for text, Base64 and Quoted-Printable transfer
|
||||
content encodings, the breaking of text into lines, SMTP
|
||||
dot-stuffing, and there are many others. Filters become
|
||||
even more powerful when we allow them to be chained together
|
||||
to create composite filters. In this context, filters can be
|
||||
seen as the internal links in a chain of data transformations.
|
||||
Sources and sinks are the corresponding end points in these
|
||||
chains. A source is a function that produces data, chunk by
|
||||
chunk, and a sink is a function that takes data, chunk by
|
||||
chunk. Finally, pumps are procedures that actively drive
|
||||
data from a source to a sink, and indirectly through all
|
||||
intervening filters. In this article, we describe the design of an
|
||||
elegant interface for filters, sources, sinks, chains, and
|
||||
pumps, and we illustrate each step with concrete examples.
|
||||
\end{abstract}
|
||||
|
||||
|
||||
\section{Introduction}
|
||||
|
||||
Within the realm of networking applications, we are often
|
||||
required apply transformations to streams of data. Examples
|
||||
required to apply transformations to streams of data. Examples
|
||||
include the end-of-line normalization for text, Base64 and
|
||||
Quoted-Printable transfer content encodings, breaking text
|
||||
into lines with a maximum number of columns, SMTP
|
||||
|
|
@ -50,11 +54,10 @@ transfer coding, and the list goes on.
|
|||
Many complex tasks require a combination of two or more such
|
||||
transformations, and therefore a general mechanism for
|
||||
promoting reuse is desirable. In the process of designing
|
||||
\texttt{LuaSocket~2.0}, David Burgess and I were forced to deal with
|
||||
this problem. The solution we reached proved to be very
|
||||
general and convenient. It is based on the concepts of
|
||||
filters, sources, sinks, and pumps, which we introduce
|
||||
below.
|
||||
\texttt{LuaSocket~2.0}, we repeatedly faced this problem.
|
||||
The solution we reached proved to be very general and
|
||||
convenient. It is based on the concepts of filters, sources,
|
||||
sinks, and pumps, which we introduce below.
|
||||
|
||||
\emph{Filters} are functions that can be repeatedly invoked
|
||||
with chunks of input, successively returning processed
|
||||
|
|
@ -62,34 +65,33 @@ chunks of output. More importantly, the result of
|
|||
concatenating all the output chunks must be the same as the
|
||||
result of applying the filter to the concatenation of all
|
||||
input chunks. In fancier language, filters \emph{commute}
|
||||
with the concatenation operator. As a result, chunk
|
||||
boundaries are irrelevant: filters correctly handle input
|
||||
data no matter how it is split.
|
||||
with the concatenation operator. More importantly, filters
|
||||
must handle input data correctly no matter how the stream
|
||||
has been split into chunks.
|
||||
|
||||
A \emph{chain} transparently combines the effect of one or
|
||||
more filters. The interface of a chain is
|
||||
indistinguishable from the interface of its components.
|
||||
This allows a chained filter to be used wherever an atomic
|
||||
filter is expected. In particular, chains can be
|
||||
A \emph{chain} is a function that transparently combines the
|
||||
effect of one or more filters. The interface of a chain is
|
||||
indistinguishable from the interface of its component
|
||||
filters. This allows a chained filter to be used wherever
|
||||
an atomic filter is accepted. In particular, chains can be
|
||||
themselves chained to create arbitrarily complex operations.
|
||||
|
||||
Filters can be seen as internal nodes in a network through
|
||||
which data will flow, potentially being transformed many
|
||||
times along its way. Chains connect these nodes together.
|
||||
To complete the picture, we need \emph{sources} and
|
||||
\emph{sinks}. These are the initial and final nodes of the
|
||||
network, respectively. Less abstractly, a source is a
|
||||
function that produces new data every time it is called.
|
||||
Conversely, sinks are functions that give a final
|
||||
destination to the data they receive. Naturally, sources
|
||||
and sinks can also be chained with filters to produce
|
||||
filtered sources and sinks.
|
||||
times along the way. Chains connect these nodes together.
|
||||
The initial and final nodes of the network are
|
||||
\emph{sources} and \emph{sinks}, respectively. Less
|
||||
abstractly, a source is a function that produces new data
|
||||
every time it is invoked. Conversely, sinks are functions
|
||||
that give a final destination to the data they receive.
|
||||
Naturally, sources and sinks can also be chained with
|
||||
filters to produce filtered sources and sinks.
|
||||
|
||||
Finally, filters, chains, sources, and sinks are all passive
|
||||
entities: they must be repeatedly invoked in order for
|
||||
anything to happen. \emph{Pumps} provide the driving force
|
||||
that pushes data through the network, from a source to a
|
||||
sink.
|
||||
sink, and indirectly through all intervening filters.
|
||||
|
||||
In the following sections, we start with a simplified
|
||||
interface, which we later refine. The evolution we present
|
||||
|
|
@ -99,27 +101,28 @@ concepts within our application domain.
|
|||
|
||||
\subsection{A simple example}
|
||||
|
||||
Let us use the end-of-line normalization of text as an
|
||||
The end-of-line normalization of text is a good
|
||||
example to motivate our initial filter interface.
|
||||
Assume we are given text in an unknown end-of-line
|
||||
convention (including possibly mixed conventions) out of the
|
||||
commonly found Unix (LF), Mac OS (CR), and DOS (CRLF)
|
||||
conventions. We would like to be able to write code like the
|
||||
following:
|
||||
commonly found Unix (\LF), Mac OS (\CR), and
|
||||
DOS (\CRLF) conventions. We would like to be able to
|
||||
use the folowing code to normalize the end-of-line markers:
|
||||
\begin{quote}
|
||||
\begin{lua}
|
||||
@stick#
|
||||
local in = source.chain(source.file(io.stdin), normalize("\r\n"))
|
||||
local out = sink.file(io.stdout)
|
||||
pump.all(in, out)
|
||||
local CRLF = "\013\010"
|
||||
local input = source.chain(source.file(io.stdin), normalize(CRLF))
|
||||
local output = sink.file(io.stdout)
|
||||
pump.all(input, output)
|
||||
%
|
||||
\end{lua}
|
||||
\end{quote}
|
||||
|
||||
This program should read data from the standard input stream
|
||||
and normalize the end-of-line markers to the canonic CRLF
|
||||
marker, as defined by the MIME standard. Finally, the
|
||||
normalized text should be sent to the standard output
|
||||
and normalize the end-of-line markers to the canonic
|
||||
\CRLF\ marker, as defined by the MIME standard.
|
||||
Finally, the normalized text should be sent to the standard output
|
||||
stream. We use a \emph{file source} that produces data from
|
||||
standard input, and chain it with a filter that normalizes
|
||||
the data. The pump then repeatedly obtains data from the
|
||||
|
|
@ -127,27 +130,28 @@ source, and passes it to the \emph{file sink}, which sends
|
|||
it to the standard output.
|
||||
|
||||
In the code above, the \texttt{normalize} \emph{factory} is a
|
||||
function that creates our normalization filter. This filter
|
||||
will replace any end-of-line marker with the canonic
|
||||
`\verb|\r\n|' marker. The initial filter interface is
|
||||
function that creates our normalization filter, which
|
||||
replaces any end-of-line marker with the canonic marker.
|
||||
The initial filter interface is
|
||||
trivial: a filter function receives a chunk of input data,
|
||||
and returns a chunk of processed data. When there are no
|
||||
more input data left, the caller notifies the filter by invoking
|
||||
it with a \texttt{nil} chunk. The filter responds by returning
|
||||
the final chunk of processed data.
|
||||
it with a \nil\ chunk. The filter responds by returning
|
||||
the final chunk of processed data (which could of course be
|
||||
the empty string).
|
||||
|
||||
Although the interface is extremely simple, the
|
||||
implementation is not so obvious. A normalization filter
|
||||
respecting this interface needs to keep some kind of context
|
||||
between calls. This is because a chunk boundary may lie between
|
||||
the CR and LF characters marking the end of a line. This
|
||||
the \CR\ and \LF\ characters marking the end of a single line. This
|
||||
need for contextual storage motivates the use of
|
||||
factories: each time the factory is invoked, it returns a
|
||||
filter with its own context so that we can have several
|
||||
independent filters being used at the same time. For
|
||||
efficiency reasons, we must avoid the obvious solution of
|
||||
concatenating all the input into the context before
|
||||
producing any output.
|
||||
producing any output chunks.
|
||||
|
||||
To that end, we break the implementation into two parts:
|
||||
a low-level filter, and a factory of high-level filters. The
|
||||
|
|
@ -167,10 +171,10 @@ end-of-line normalization filters:
|
|||
\begin{quote}
|
||||
\begin{lua}
|
||||
@stick#
|
||||
function filter.cycle(low, ctx, extra)
|
||||
function filter.cycle(lowlevel, context, extra)
|
||||
return function(chunk)
|
||||
local ret
|
||||
ret, ctx = low(ctx, chunk, extra)
|
||||
ret, context = lowlevel(context, chunk, extra)
|
||||
return ret
|
||||
end
|
||||
end
|
||||
|
|
@ -178,27 +182,30 @@ end
|
|||
|
||||
@stick#
|
||||
function normalize(marker)
|
||||
return cycle(eol, 0, marker)
|
||||
return filter.cycle(eol, 0, marker)
|
||||
end
|
||||
%
|
||||
\end{lua}
|
||||
\end{quote}
|
||||
|
||||
The \texttt{normalize} factory simply calls a more generic
|
||||
factory, the \texttt{cycle} factory. This factory receives a
|
||||
factory, the \texttt{cycle}~factory, passing the low-level
|
||||
filter~\texttt{eol}. The \texttt{cycle}~factory receives a
|
||||
low-level filter, an initial context, and an extra
|
||||
parameter, and returns a new high-level filter. Each time
|
||||
the high-level filer is passed a new chunk, it invokes the
|
||||
low-level filter with the previous context, the new chunk,
|
||||
and the extra argument. It is the low-level filter that
|
||||
does all the work, producing the chunk of processed data and
|
||||
a new context. The high-level filter then updates its
|
||||
a new context. The high-level filter then replaces its
|
||||
internal context, and returns the processed chunk of data to
|
||||
the user. Notice that we take advantage of Lua's lexical
|
||||
scoping to store the context in a closure between function
|
||||
calls.
|
||||
|
||||
Concerning the low-level filter code, we must first accept
|
||||
\subsection{The C part of the filter}
|
||||
|
||||
As for the low-level filter, we must first accept
|
||||
that there is no perfect solution to the end-of-line marker
|
||||
normalization problem. The difficulty comes from an
|
||||
inherent ambiguity in the definition of empty lines within
|
||||
|
|
@ -208,39 +215,39 @@ mixed input. It also does a reasonable job with empty lines
|
|||
and serves as a good example of how to implement a low-level
|
||||
filter.
|
||||
|
||||
The idea is to consider both CR and~LF as end-of-line
|
||||
The idea is to consider both \CR\ and~\LF\ as end-of-line
|
||||
\emph{candidates}. We issue a single break if any candidate
|
||||
is seen alone, or followed by a different candidate. In
|
||||
other words, CR~CR~and LF~LF each issue two end-of-line
|
||||
markers, whereas CR~LF~and LF~CR issue only one marker each.
|
||||
This method correctly handles the Unix, DOS/MIME, VMS, and Mac
|
||||
OS conventions.
|
||||
is seen alone, or if it is followed by a different
|
||||
candidate. In other words, \CR~\CR~and \LF~\LF\ each issue
|
||||
two end-of-line markers, whereas \CR~\LF~and \LF~\CR\ issue
|
||||
only one marker each. It is easy to see that this method
|
||||
correctly handles the most common end-of-line conventions.
|
||||
|
||||
\subsection{The C part of the filter}
|
||||
|
||||
Our low-level filter is divided into two simple functions.
|
||||
The inner function performs the normalization itself. It takes
|
||||
each input character in turn, deciding what to output and
|
||||
how to modify the context. The context tells if the last
|
||||
processed character was an end-of-line candidate, and if so,
|
||||
which candidate it was. For efficiency, it uses
|
||||
Lua's auxiliary library's buffer interface:
|
||||
With this in mind, we divide the low-level filter into two
|
||||
simple functions. The inner function~\texttt{pushchar} performs the
|
||||
normalization itself. It takes each input character in turn,
|
||||
deciding what to output and how to modify the context. The
|
||||
context tells if the last processed character was an
|
||||
end-of-line candidate, and if so, which candidate it was.
|
||||
For efficiency, we use Lua's auxiliary library's buffer
|
||||
interface:
|
||||
\begin{quote}
|
||||
\begin{C}
|
||||
@stick#
|
||||
@#define candidate(c) (c == CR || c == LF)
|
||||
static int process(int c, int last, const char *marker,
|
||||
static int pushchar(int c, int last, const char *marker,
|
||||
luaL_Buffer *buffer) {
|
||||
if (candidate(c)) {
|
||||
if (candidate(last)) {
|
||||
if (c == last) luaL_addstring(buffer, marker);
|
||||
if (c == last)
|
||||
luaL_addstring(buffer, marker);
|
||||
return 0;
|
||||
} else {
|
||||
luaL_addstring(buffer, marker);
|
||||
return c;
|
||||
}
|
||||
} else {
|
||||
luaL_putchar(buffer, c);
|
||||
luaL_pushchar(buffer, c);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -248,15 +255,20 @@ static int process(int c, int last, const char *marker,
|
|||
\end{C}
|
||||
\end{quote}
|
||||
|
||||
The outer function simply interfaces with Lua. It receives the
|
||||
context and input chunk (as well as an optional
|
||||
custom end-of-line marker), and returns the transformed
|
||||
output chunk and the new context:
|
||||
The outer function~\texttt{eol} simply interfaces with Lua.
|
||||
It receives the context and input chunk (as well as an
|
||||
optional custom end-of-line marker), and returns the
|
||||
transformed output chunk and the new context.
|
||||
Notice that if the input chunk is \nil, the operation
|
||||
is considered to be finished. In that case, the loop will
|
||||
not execute a single time and the context is reset to the
|
||||
initial state. This allows the filter to be reused many
|
||||
times:
|
||||
\begin{quote}
|
||||
\begin{C}
|
||||
@stick#
|
||||
static int eol(lua_State *L) {
|
||||
int ctx = luaL_checkint(L, 1);
|
||||
int context = luaL_checkint(L, 1);
|
||||
size_t isize = 0;
|
||||
const char *input = luaL_optlstring(L, 2, NULL, &isize);
|
||||
const char *last = input + isize;
|
||||
|
|
@ -269,24 +281,18 @@ static int eol(lua_State *L) {
|
|||
return 2;
|
||||
}
|
||||
while (input < last)
|
||||
ctx = process(*input++, ctx, marker, &buffer);
|
||||
context = pushchar(*input++, context, marker, &buffer);
|
||||
luaL_pushresult(&buffer);
|
||||
lua_pushnumber(L, ctx);
|
||||
lua_pushnumber(L, context);
|
||||
return 2;
|
||||
}
|
||||
%
|
||||
\end{C}
|
||||
\end{quote}
|
||||
|
||||
Notice that if the input chunk is \texttt{nil}, the operation
|
||||
is considered to be finished. In that case, the loop will
|
||||
not execute a single time and the context is reset to the
|
||||
initial state. This allows the filter to be reused many
|
||||
times.
|
||||
|
||||
When designing your own filters, the challenging part is to
|
||||
decide what will be in the context. For line breaking, for
|
||||
instance, it could be the number of bytes left in the
|
||||
instance, it could be the number of bytes that still fit in the
|
||||
current line. For Base64 encoding, it could be a string
|
||||
with the bytes that remain after the division of the input
|
||||
into 3-byte atoms. The MIME module in the \texttt{LuaSocket}
|
||||
|
|
@ -294,19 +300,22 @@ distribution has many other examples.
|
|||
|
||||
\section{Filter chains}
|
||||
|
||||
Chains add a lot to the power of filters. For example,
|
||||
Chains greatly increase the power of filters. For example,
|
||||
according to the standard for Quoted-Printable encoding,
|
||||
text must be normalized to a canonic end-of-line marker
|
||||
prior to encoding. To help specifying complex
|
||||
transformations like this, we define a chain factory that
|
||||
creates a composite filter from one or more filters. A
|
||||
chained filter passes data through all its components, and
|
||||
can be used wherever a primitive filter is accepted.
|
||||
text should be normalized to a canonic end-of-line marker
|
||||
prior to encoding. After encoding, the resulting text must
|
||||
be broken into lines of no more than 76 characters, with the
|
||||
use of soft line breaks (a line terminated by the \texttt{=}
|
||||
sign). To help specifying complex transformations like
|
||||
this, we define a chain factory that creates a composite
|
||||
filter from one or more filters. A chained filter passes
|
||||
data through all its components, and can be used wherever a
|
||||
primitive filter is accepted.
|
||||
|
||||
The chaining factory is very simple. The auxiliary
|
||||
function~\texttt{chainpair} chains two filters together,
|
||||
taking special care if the chunk is the last. This is
|
||||
because the final \texttt{nil} chunk notification has to be
|
||||
because the final \nil\ chunk notification has to be
|
||||
pushed through both filters in turn:
|
||||
\begin{quote}
|
||||
\begin{lua}
|
||||
|
|
@ -322,9 +331,9 @@ end
|
|||
|
||||
@stick#
|
||||
function filter.chain(...)
|
||||
local f = arg[1]
|
||||
for i = 2, @#arg do
|
||||
f = chainpair(f, arg[i])
|
||||
local f = select(1, ...)
|
||||
for i = 2, select('@#', ...) do
|
||||
f = chainpair(f, select(i, ...))
|
||||
end
|
||||
return f
|
||||
end
|
||||
|
|
@ -337,11 +346,11 @@ define the Quoted-Printable conversion as such:
|
|||
\begin{quote}
|
||||
\begin{lua}
|
||||
@stick#
|
||||
local qp = filter.chain(normalize("\r\n"),
|
||||
encode("quoted-printable"))
|
||||
local in = source.chain(source.file(io.stdin), qp)
|
||||
local out = sink.file(io.stdout)
|
||||
pump.all(in, out)
|
||||
local qp = filter.chain(normalize(CRLF), encode("quoted-printable"),
|
||||
wrap("quoted-printable"))
|
||||
local input = source.chain(source.file(io.stdin), qp)
|
||||
local output = sink.file(io.stdout)
|
||||
pump.all(input, output)
|
||||
%
|
||||
\end{lua}
|
||||
\end{quote}
|
||||
|
|
@ -360,14 +369,14 @@ gives a final destination to the data.
|
|||
\subsection{Sources}
|
||||
|
||||
A source returns the next chunk of data each time it is
|
||||
invoked. When there is no more data, it simply returns
|
||||
\texttt{nil}. In the event of an error, the source can inform the
|
||||
caller by returning \texttt{nil} followed by an error message.
|
||||
invoked. When there is no more data, it simply returns~\nil.
|
||||
In the event of an error, the source can inform the
|
||||
caller by returning \nil\ followed by the error message.
|
||||
|
||||
Below are two simple source factories. The \texttt{empty} source
|
||||
returns no data, possibly returning an associated error
|
||||
message. The \texttt{file} source works harder, and
|
||||
yields the contents of a file in a chunk by chunk fashion:
|
||||
message. The \texttt{file} source yields the contents of a file
|
||||
in a chunk by chunk fashion:
|
||||
\begin{quote}
|
||||
\begin{lua}
|
||||
@stick#
|
||||
|
|
@ -398,7 +407,7 @@ A filtered source passes its data through the
|
|||
associated filter before returning it to the caller.
|
||||
Filtered sources are useful when working with
|
||||
functions that get their input data from a source (such as
|
||||
the pump in our first example). By chaining a source with one or
|
||||
the pumps in our examples). By chaining a source with one or
|
||||
more filters, the function can be transparently provided
|
||||
with filtered data, with no need to change its interface.
|
||||
Here is a factory that does the job:
|
||||
|
|
@ -406,14 +415,18 @@ Here is a factory that does the job:
|
|||
\begin{lua}
|
||||
@stick#
|
||||
function source.chain(src, f)
|
||||
return source.simplify(function()
|
||||
if not src then return nil end
|
||||
return function()
|
||||
if not src then
|
||||
return nil
|
||||
end
|
||||
local chunk, err = src()
|
||||
if not chunk then
|
||||
src = nil
|
||||
return f(nil)
|
||||
else return f(chunk) end
|
||||
end)
|
||||
else
|
||||
return f(chunk)
|
||||
end
|
||||
end
|
||||
end
|
||||
%
|
||||
\end{lua}
|
||||
|
|
@ -421,20 +434,20 @@ end
|
|||
|
||||
\subsection{Sinks}
|
||||
|
||||
Just as we defined an interface a data source,
|
||||
Just as we defined an interface for source of data,
|
||||
we can also define an interface for a data destination.
|
||||
We call any function respecting this
|
||||
interface a \emph{sink}. In our first example, we used a
|
||||
file sink connected to the standard output.
|
||||
|
||||
Sinks receive consecutive chunks of data, until the end of
|
||||
data is signaled by a \texttt{nil} chunk. A sink can be
|
||||
data is signaled by a \nil\ input chunk. A sink can be
|
||||
notified of an error with an optional extra argument that
|
||||
contains the error message, following a \texttt{nil} chunk.
|
||||
contains the error message, following a \nil\ chunk.
|
||||
If a sink detects an error itself, and
|
||||
wishes not to be called again, it can return \texttt{nil},
|
||||
wishes not to be called again, it can return \nil,
|
||||
followed by an error message. A return value that
|
||||
is not \texttt{nil} means the source will accept more data.
|
||||
is not \nil\ means the sink will accept more data.
|
||||
|
||||
Below are two useful sink factories.
|
||||
The table factory creates a sink that stores
|
||||
|
|
@ -469,7 +482,7 @@ end
|
|||
|
||||
Naturally, filtered sinks are just as useful as filtered
|
||||
sources. A filtered sink passes each chunk it receives
|
||||
through the associated filter before handing it to the
|
||||
through the associated filter before handing it down to the
|
||||
original sink. In the following example, we use a source
|
||||
that reads from the standard input. The input chunks are
|
||||
sent to a table sink, which has been coupled with a
|
||||
|
|
@ -479,10 +492,10 @@ standard out:
|
|||
\begin{quote}
|
||||
\begin{lua}
|
||||
@stick#
|
||||
local in = source.file(io.stdin)
|
||||
local out, t = sink.table()
|
||||
out = sink.chain(normalize("\r\n"), out)
|
||||
pump.all(in, out)
|
||||
local input = source.file(io.stdin)
|
||||
local output, t = sink.table()
|
||||
output = sink.chain(normalize(CRLF), output)
|
||||
pump.all(input, output)
|
||||
io.write(table.concat(t))
|
||||
%
|
||||
\end{lua}
|
||||
|
|
@ -490,11 +503,11 @@ io.write(table.concat(t))
|
|||
|
||||
\subsection{Pumps}
|
||||
|
||||
Adrian Sietsma noticed that, although not on purpose, our
|
||||
interface for sources is compatible with Lua iterators.
|
||||
That is, a source can be neatly used in conjunction
|
||||
with \texttt{for} loops. Using our file
|
||||
source as an iterator, we can write the following code:
|
||||
Although not on purpose, our interface for sources is
|
||||
compatible with Lua iterators. That is, a source can be
|
||||
neatly used in conjunction with \texttt{for} loops. Using
|
||||
our file source as an iterator, we can write the following
|
||||
code:
|
||||
\begin{quote}
|
||||
\begin{lua}
|
||||
@stick#
|
||||
|
|
@ -539,20 +552,22 @@ end
|
|||
The \texttt{pump.step} function moves one chunk of data from
|
||||
the source to the sink. The \texttt{pump.all} function takes
|
||||
an optional \texttt{step} function and uses it to pump all the
|
||||
data from the source to the sink. We can now use everything
|
||||
we have to write a program that reads a binary file from
|
||||
data from the source to the sink.
|
||||
Here is an example that uses the Base64 and the
|
||||
line wrapping filters from the \texttt{LuaSocket}
|
||||
distribution. The program reads a binary file from
|
||||
disk and stores it in another file, after encoding it to the
|
||||
Base64 transfer content encoding:
|
||||
\begin{quote}
|
||||
\begin{lua}
|
||||
@stick#
|
||||
local in = source.chain(
|
||||
local input = source.chain(
|
||||
source.file(io.open("input.bin", "rb")),
|
||||
encode("base64"))
|
||||
local out = sink.chain(
|
||||
local output = sink.chain(
|
||||
wrap(76),
|
||||
sink.file(io.open("output.b64", "w")))
|
||||
pump.all(in, out)
|
||||
pump.all(input, output)
|
||||
%
|
||||
\end{lua}
|
||||
\end{quote}
|
||||
|
|
@ -561,19 +576,17 @@ The way we split the filters here is not intuitive, on
|
|||
purpose. Alternatively, we could have chained the Base64
|
||||
encode filter and the line-wrap filter together, and then
|
||||
chain the resulting filter with either the file source or
|
||||
the file sink. It doesn't really matter. The Base64 and the
|
||||
line wrapping filters are part of the \texttt{LuaSocket}
|
||||
distribution.
|
||||
the file sink. It doesn't really matter.
|
||||
|
||||
\section{Exploding filters}
|
||||
|
||||
Our current filter interface has one flagrant shortcoming.
|
||||
When David Burgess was writing his \texttt{gzip} filter, he
|
||||
noticed that a decompression filter can explode a small
|
||||
input chunk into a huge amount of data. To address this
|
||||
problem, we decided to change the filter interface and allow
|
||||
exploding filters to return large quantities of output data
|
||||
in a chunk by chunk manner.
|
||||
Our current filter interface has one serious shortcoming.
|
||||
Consider for example a \texttt{gzip} decompression filter.
|
||||
During decompression, a small input chunk can be exploded
|
||||
into a huge amount of data. To address this problem, we
|
||||
decided to change the filter interface and allow exploding
|
||||
filters to return large quantities of output data in a chunk
|
||||
by chunk manner.
|
||||
|
||||
More specifically, after passing each chunk of input to
|
||||
a filter, and collecting the first chunk of output, the
|
||||
|
|
@ -582,11 +595,11 @@ filtered data is left. Within these secondary calls, the
|
|||
caller passes an empty string to the filter. The filter
|
||||
responds with an empty string when it is ready for the next
|
||||
input chunk. In the end, after the user passes a
|
||||
\texttt{nil} chunk notifying the filter that there is no
|
||||
\nil\ chunk notifying the filter that there is no
|
||||
more input data, the filter might still have to produce too
|
||||
much output data to return in a single chunk. The user has
|
||||
to loop again, now passing \texttt{nil} to the filter each time,
|
||||
until the filter itself returns \texttt{nil} to notify the
|
||||
to loop again, now passing \nil\ to the filter each time,
|
||||
until the filter itself returns \nil\ to notify the
|
||||
user it is finally done.
|
||||
|
||||
Fortunately, it is very easy to modify a filter to respect
|
||||
|
|
@ -599,13 +612,13 @@ Interestingly, the modifications do not have a measurable
|
|||
negative impact in the performance of filters that do
|
||||
not need the added flexibility. On the other hand, for a
|
||||
small price in complexity, the changes make exploding
|
||||
filters practical.
|
||||
filters practical.
|
||||
|
||||
\section{A complex example}
|
||||
|
||||
The LTN12 module in the \texttt{LuaSocket} distribution
|
||||
implements the ideas we have described. The MIME
|
||||
and SMTP modules are especially integrated with LTN12,
|
||||
implements all the ideas we have described. The MIME
|
||||
and SMTP modules are tightly integrated with LTN12,
|
||||
and can be used to showcase the expressive power of filters,
|
||||
sources, sinks, and pumps. Below is an example
|
||||
of how a user would proceed to define and send a
|
||||
|
|
@ -622,9 +635,9 @@ local message = smtp.message{
|
|||
to = "Fulano <fulano@example.com>",
|
||||
subject = "A message with an attachment"},
|
||||
body = {
|
||||
preamble = "Hope you can see the attachment\r\n",
|
||||
preamble = "Hope you can see the attachment" .. CRLF,
|
||||
[1] = {
|
||||
body = "Here is our logo\r\n"},
|
||||
body = "Here is our logo" .. CRLF},
|
||||
[2] = {
|
||||
headers = {
|
||||
["content-type"] = 'image/png; name="luasocket.png"',
|
||||
|
|
@ -665,6 +678,18 @@ abstraction for final data destinations. Filters define an
|
|||
interface for data transformations. The chaining of
|
||||
filters, sources and sinks provides an elegant way to create
|
||||
arbitrarily complex data transformations from simpler
|
||||
components. Pumps simply move the data through.
|
||||
components. Pumps simply push the data through.
|
||||
|
||||
\section{Acknowledgements}
|
||||
|
||||
The concepts described in this text are the result of long
|
||||
discussions with David Burgess. A version of this text has
|
||||
been released on-line as the Lua Technical Note 012, hence
|
||||
the name of the corresponding LuaSocket module,
|
||||
\texttt{ltn12}. Wim Couwenberg contributed to the
|
||||
implementation of the module, and Adrian Sietsma was the
|
||||
first to notice the correspondence between sources and Lua
|
||||
iterators.
|
||||
|
||||
|
||||
\end{document}
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
|
|
@ -12,3 +12,12 @@ clean:
|
|||
|
||||
pdf: ltn012.pdf
|
||||
open ltn012.pdf
|
||||
|
||||
test: gem.so
|
||||
|
||||
|
||||
gem.o: gem.c
|
||||
gcc -c -o gem.o -Wall -ansi -W -O2 gem.c
|
||||
|
||||
gem.so: gem.o
|
||||
export MACOSX_DEPLOYMENT_TARGET="10.3"; gcc -bundle -undefined dynamic_lookup -o gem.so gem.o
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
source = {}
|
||||
sink = {}
|
||||
pump = {}
|
||||
filter = {}
|
||||
|
||||
-- source.chain
|
||||
dofile("ex6.lua")
|
||||
|
||||
-- source.file
|
||||
dofile("ex5.lua")
|
||||
|
||||
-- normalize
|
||||
require"gem"
|
||||
eol = gem.eol
|
||||
dofile("ex2.lua")
|
||||
|
||||
-- sink.file
|
||||
require"ltn12"
|
||||
sink.file = ltn12.sink.file
|
||||
|
||||
-- pump.all
|
||||
dofile("ex10.lua")
|
||||
|
||||
-- run test
|
||||
dofile("ex1.lua")
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
this is a test file
|
||||
it should have been saved as lf eol
|
||||
but t1.lua will convert it to crlf eol
|
||||
otherwise it is broken!
|
||||
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
source = {}
|
||||
sink = {}
|
||||
pump = {}
|
||||
filter = {}
|
||||
|
||||
-- filter.chain
|
||||
dofile("ex3.lua")
|
||||
|
||||
-- normalize
|
||||
require"gem"
|
||||
eol = gem.eol
|
||||
dofile("ex2.lua")
|
||||
|
||||
-- encode
|
||||
require"mime"
|
||||
encode = mime.encode
|
||||
|
||||
-- wrap
|
||||
wrap = mime.wrap
|
||||
|
||||
-- source.chain
|
||||
dofile("ex6.lua")
|
||||
|
||||
-- source.file
|
||||
dofile("ex5.lua")
|
||||
|
||||
-- sink.file
|
||||
require"ltn12"
|
||||
sink.file = ltn12.sink.file
|
||||
|
||||
-- pump.all
|
||||
dofile("ex10.lua")
|
||||
|
||||
-- run test
|
||||
CRLF = "\013\010"
|
||||
dofile("ex4.lua")
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
esse é um texto com acentos
|
||||
quoted-printable tem que quebrar linhas longas, com mais que 76 linhas de texto
|
||||
fora que as quebras de linhas têm que ser normalizadas
|
||||
vamos ver o que dá isso aqui
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
esse =E9 um texto com acentos
|
||||
quoted-printable tem que quebrar linhas longas, com mais que 76 linhas de t=
|
||||
exto
|
||||
fora que as quebras de linhas t=EAm que ser normalizadas
|
||||
vamos ver o que d=E1 isso aqui
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
source = {}
|
||||
sink = {}
|
||||
pump = {}
|
||||
filter = {}
|
||||
|
||||
-- source.file
|
||||
dofile("ex5.lua")
|
||||
|
||||
-- sink.table
|
||||
dofile("ex7.lua")
|
||||
|
||||
-- sink.chain
|
||||
require"ltn12"
|
||||
sink.chain = ltn12.sink.chain
|
||||
|
||||
-- normalize
|
||||
require"gem"
|
||||
eol = gem.eol
|
||||
dofile("ex2.lua")
|
||||
|
||||
-- pump.all
|
||||
dofile("ex10.lua")
|
||||
|
||||
-- run test
|
||||
dofile("ex8.lua")
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
source = {}
|
||||
sink = {}
|
||||
pump = {}
|
||||
filter = {}
|
||||
|
||||
-- source.file
|
||||
dofile("ex5.lua")
|
||||
|
||||
-- run test
|
||||
dofile("ex9.lua")
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
source = {}
|
||||
sink = {}
|
||||
pump = {}
|
||||
filter = {}
|
||||
|
||||
-- source.chain
|
||||
dofile("ex6.lua")
|
||||
|
||||
-- source.file
|
||||
dofile("ex5.lua")
|
||||
|
||||
-- encode
|
||||
require"mime"
|
||||
encode = mime.encode
|
||||
|
||||
-- sink.chain
|
||||
require"ltn12"
|
||||
sink.chain = ltn12.sink.chain
|
||||
|
||||
-- wrap
|
||||
wrap = mime.wrap
|
||||
|
||||
-- sink.file
|
||||
sink.file = ltn12.sink.file
|
||||
|
||||
-- pump.all
|
||||
dofile("ex10.lua")
|
||||
|
||||
-- run test
|
||||
dofile("ex11.lua")
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
function readfile(n)
|
||||
local f = io.open(n, "rb")
|
||||
local s = f:read("*a")
|
||||
f:close()
|
||||
return s
|
||||
end
|
||||
|
||||
lf = readfile("t1lf.txt")
|
||||
os.remove("t1crlf.txt")
|
||||
os.execute("lua t1.lua < t1lf.txt > t1crlf.txt")
|
||||
crlf = readfile("t1crlf.txt")
|
||||
assert(crlf == string.gsub(lf, "\010", "\013\010"), "broken")
|
||||
|
||||
gt = readfile("t2gt.qp")
|
||||
os.remove("t2.qp")
|
||||
os.execute("lua t2.lua < t2.txt > t2.qp")
|
||||
t2 = readfile("t2.qp")
|
||||
assert(gt == t2, "broken")
|
||||
|
||||
os.remove("t1crlf.txt")
|
||||
os.execute("lua t3.lua < t1lf.txt > t1crlf.txt")
|
||||
crlf = readfile("t1crlf.txt")
|
||||
assert(crlf == string.gsub(lf, "\010", "\013\010"), "broken")
|
||||
|
||||
t = readfile("test.lua")
|
||||
os.execute("lua t4.lua < test.lua > t")
|
||||
t2 = readfile("t")
|
||||
assert(t == t2, "broken")
|
||||
|
||||
os.remove("output.b64")
|
||||
gt = readfile("gt.b64")
|
||||
os.execute("lua t5.lua")
|
||||
t5 = readfile("output.b64")
|
||||
assert(gt == t5, "failed")
|
||||
|
||||
print("1 2 5 6 10 passed")
|
||||
print("2 3 4 5 6 10 passed")
|
||||
print("2 5 6 7 8 10 passed")
|
||||
print("5 9 passed")
|
||||
print("5 6 10 11 passed")
|
||||
|
||||
os.remove("t")
|
||||
os.remove("t2.qp")
|
||||
os.remove("t1crlf.txt")
|
||||
os.remove("t11.b64")
|
||||
os.remove("output.b64")
|
||||
Loading…
Reference in New Issue