417 lines
12 KiB
HTML
417 lines
12 KiB
HTML
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
|
|
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
|
<html>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
|
|
<head>
|
|
<title>Penlight Documentation</title>
|
|
<link rel="stylesheet" href="../ldoc.css" type="text/css" />
|
|
</head>
|
|
<body>
|
|
|
|
<div id="container">
|
|
|
|
<div id="product">
|
|
<div id="product_logo"></div>
|
|
<div id="product_name"><big><b></b></big></div>
|
|
<div id="product_description"></div>
|
|
</div> <!-- id="product" -->
|
|
|
|
|
|
<div id="main">
|
|
|
|
|
|
<!-- Menu -->
|
|
|
|
<div id="navigation">
|
|
<br/>
|
|
<h1>Penlight</h1>
|
|
|
|
<ul>
|
|
<li><a href="../index.html">Index</a></li>
|
|
</ul>
|
|
|
|
<h2>Contents</h2>
|
|
<ul>
|
|
<li><a href="#Functions">Functions</a></li>
|
|
</ul>
|
|
|
|
|
|
<h2>Modules</h2>
|
|
<ul>
|
|
<li><a href="../modules/pl.html">pl</a></li>
|
|
<li><a href="../modules/pl.Date.html">pl.Date</a></li>
|
|
<li><a href="../modules/pl.List.html">pl.List</a></li>
|
|
<li><a href="../modules/pl.Map.html">pl.Map</a></li>
|
|
<li><a href="../modules/pl.MultiMap.html">pl.MultiMap</a></li>
|
|
<li><a href="../modules/pl.OrderedMap.html">pl.OrderedMap</a></li>
|
|
<li><a href="../modules/pl.Set.html">pl.Set</a></li>
|
|
<li><a href="../modules/pl.app.html">pl.app</a></li>
|
|
<li><a href="../modules/pl.array2d.html">pl.array2d</a></li>
|
|
<li><a href="../modules/pl.class.html">pl.class</a></li>
|
|
<li><a href="../modules/pl.comprehension.html">pl.comprehension</a></li>
|
|
<li><a href="../modules/pl.config.html">pl.config</a></li>
|
|
<li><a href="../modules/pl.data.html">pl.data</a></li>
|
|
<li><a href="../modules/pl.dir.html">pl.dir</a></li>
|
|
<li><a href="../modules/pl.file.html">pl.file</a></li>
|
|
<li><a href="../modules/pl.func.html">pl.func</a></li>
|
|
<li><a href="../modules/pl.input.html">pl.input</a></li>
|
|
<li><a href="../modules/pl.lapp.html">pl.lapp</a></li>
|
|
<li><strong>pl.lexer</strong></li>
|
|
<li><a href="../modules/pl.luabalanced.html">pl.luabalanced</a></li>
|
|
<li><a href="../modules/pl.operator.html">pl.operator</a></li>
|
|
<li><a href="../modules/pl.path.html">pl.path</a></li>
|
|
<li><a href="../modules/pl.permute.html">pl.permute</a></li>
|
|
<li><a href="../modules/pl.pretty.html">pl.pretty</a></li>
|
|
<li><a href="../modules/pl.seq.html">pl.seq</a></li>
|
|
<li><a href="../modules/pl.sip.html">pl.sip</a></li>
|
|
<li><a href="../modules/pl.strict.html">pl.strict</a></li>
|
|
<li><a href="../modules/pl.stringio.html">pl.stringio</a></li>
|
|
<li><a href="../modules/pl.stringx.html">pl.stringx</a></li>
|
|
<li><a href="../modules/pl.tablex.html">pl.tablex</a></li>
|
|
<li><a href="../modules/pl.template.html">pl.template</a></li>
|
|
<li><a href="../modules/pl.test.html">pl.test</a></li>
|
|
<li><a href="../modules/pl.text.html">pl.text</a></li>
|
|
<li><a href="../modules/pl.utils.html">pl.utils</a></li>
|
|
<li><a href="../modules/pl.xml.html">pl.xml</a></li>
|
|
</ul>
|
|
<h2>Topics</h2>
|
|
<ul>
|
|
<li><a href="../topics/01-introduction.md.html">01-introduction.md</a></li>
|
|
<li><a href="../topics/02-arrays.md.html">02-arrays.md</a></li>
|
|
<li><a href="../topics/03-strings.md.html">03-strings.md</a></li>
|
|
<li><a href="../topics/04-paths.md.html">04-paths.md</a></li>
|
|
<li><a href="../topics/05-dates.md.html">05-dates.md</a></li>
|
|
<li><a href="../topics/06-data.md.html">06-data.md</a></li>
|
|
<li><a href="../topics/07-functional.md.html">07-functional.md</a></li>
|
|
<li><a href="../topics/08-additional.md.html">08-additional.md</a></li>
|
|
<li><a href="../topics/09-discussion.md.html">09-discussion.md</a></li>
|
|
</ul>
|
|
<h2>Examples</h2>
|
|
<ul>
|
|
<li><a href="../examples/seesubst.lua.html">seesubst.lua</a></li>
|
|
<li><a href="../examples/sipscan.lua.html">sipscan.lua</a></li>
|
|
<li><a href="../examples/symbols.lua.html">symbols.lua</a></li>
|
|
<li><a href="../examples/test-cmp.lua.html">test-cmp.lua</a></li>
|
|
<li><a href="../examples/test-data.lua.html">test-data.lua</a></li>
|
|
<li><a href="../examples/test-listcallbacks.lua.html">test-listcallbacks.lua</a></li>
|
|
<li><a href="../examples/test-pretty.lua.html">test-pretty.lua</a></li>
|
|
<li><a href="../examples/test-symbols.lua.html">test-symbols.lua</a></li>
|
|
<li><a href="../examples/testapp.lua.html">testapp.lua</a></li>
|
|
<li><a href="../examples/testclone.lua.html">testclone.lua</a></li>
|
|
<li><a href="../examples/testconfig.lua.html">testconfig.lua</a></li>
|
|
<li><a href="../examples/testglobal.lua.html">testglobal.lua</a></li>
|
|
<li><a href="../examples/testinputfields.lua.html">testinputfields.lua</a></li>
|
|
<li><a href="../examples/testinputfields2.lua.html">testinputfields2.lua</a></li>
|
|
<li><a href="../examples/testxml.lua.html">testxml.lua</a></li>
|
|
<li><a href="../examples/which.lua.html">which.lua</a></li>
|
|
</ul>
|
|
|
|
</div>
|
|
|
|
<div id="content">
|
|
|
|
<h1>Module <code>pl.lexer</code></h1>
|
|
|
|
<p>Lexical scanner for creating a sequence of tokens from text.</p>
|
|
<p> <code>lexer.scan(s)</code> returns an iterator over all tokens found in the
|
|
string <code>s</code>. This iterator returns two values, a token type string
|
|
(such as ‘string’ for quoted string, ‘iden’ for identifier) and the value of the
|
|
token.</p>
|
|
|
|
<p> Versions specialized for Lua and C are available; these also handle block comments
|
|
and classify keywords as ‘keyword’ tokens. For example:</p>
|
|
|
|
<pre><code>> s = 'for i=1,n do'
|
|
> for t,v in lexer.lua(s) do print(t,v) end
|
|
keyword for
|
|
iden i
|
|
= =
|
|
number 1
|
|
, ,
|
|
iden n
|
|
keyword do
|
|
</code></pre>
|
|
|
|
<p> See the Guide for further <a href="../topics/06-data.md.html#Lexical_Scanning">discussion</a></p>
|
|
|
|
<h2><a href="#Functions">Functions</a></h2>
|
|
<table class="function_list">
|
|
<tr>
|
|
<td class="name" nowrap><a href="#scan">scan (s, matches, filter, options)</a></td>
|
|
<td class="summary">create a plain token iterator from a string or file-like object.</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="name" nowrap><a href="#insert">insert (tok, a1, a2)</a></td>
|
|
<td class="summary">insert tokens into a stream.</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="name" nowrap><a href="#getline">getline (tok)</a></td>
|
|
<td class="summary">get everything in a stream upto a newline.</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="name" nowrap><a href="#lineno">lineno (tok)</a></td>
|
|
<td class="summary">get current line number.</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="name" nowrap><a href="#getrest">getrest (tok)</a></td>
|
|
<td class="summary">get the rest of the stream.</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="name" nowrap><a href="#get_keywords">get_keywords ()</a></td>
|
|
<td class="summary">get the Lua keywords as a set-like table.</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="name" nowrap><a href="#lua">lua (s, filter, options)</a></td>
|
|
<td class="summary">create a Lua token iterator from a string or file-like object.</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="name" nowrap><a href="#cpp">cpp (s, filter, options)</a></td>
|
|
<td class="summary">create a C/C++ token iterator from a string or file-like object.</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="name" nowrap><a href="#get_separated_list">get_separated_list (tok, endtoken, delim)</a></td>
|
|
<td class="summary">get a list of parameters separated by a delimiter from a stream.</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="name" nowrap><a href="#skipws">skipws (tok)</a></td>
|
|
<td class="summary">get the next non-space token from the stream.</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="name" nowrap><a href="#expecting">expecting (tok, expected_type, no_skip_ws)</a></td>
|
|
<td class="summary">get the next token, which must be of the expected type.</td>
|
|
</tr>
|
|
</table>
|
|
|
|
<br/>
|
|
<br/>
|
|
|
|
|
|
<h2><a name="Functions"></a>Functions</h2>
|
|
|
|
<dl class="function">
|
|
<dt>
|
|
<a name = "scan"></a>
|
|
<strong>scan (s, matches, filter, options)</strong>
|
|
</dt>
|
|
<dd>
|
|
create a plain token iterator from a string or file-like object.
|
|
|
|
<h3>Parameters:</h3>
|
|
<ul>
|
|
<li><code><em>s</em></code>: the string</li>
|
|
<li><code><em>matches</em></code>: an optional match table (set of pattern-action pairs)</li>
|
|
<li><code><em>filter</em></code>: a table of token types to exclude, by default {space=true}</li>
|
|
<li><code><em>options</em></code>: a table of options; by default, {number=true,string=true},
|
|
which means convert numbers and strip string quotes.</li>
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
</dd>
|
|
<dt>
|
|
<a name = "insert"></a>
|
|
<strong>insert (tok, a1, a2)</strong>
|
|
</dt>
|
|
<dd>
|
|
insert tokens into a stream.
|
|
|
|
<h3>Parameters:</h3>
|
|
<ul>
|
|
<li><code><em>tok</em></code>: a token stream</li>
|
|
<li><code><em>a1</em></code>: a string is the type, a table is a token list and
|
|
a function is assumed to be a token-like iterator (returns type & value)</li>
|
|
<li><code><em>a2</em></code>: a string is the value</li>
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
</dd>
|
|
<dt>
|
|
<a name = "getline"></a>
|
|
<strong>getline (tok)</strong>
|
|
</dt>
|
|
<dd>
|
|
get everything in a stream upto a newline.
|
|
|
|
<h3>Parameters:</h3>
|
|
<ul>
|
|
<li><code><em>tok</em></code>: a token stream</li>
|
|
</ul>
|
|
|
|
|
|
<h3>Returns:</h3>
|
|
<ol>
|
|
a string
|
|
</ol>
|
|
|
|
|
|
</dd>
|
|
<dt>
|
|
<a name = "lineno"></a>
|
|
<strong>lineno (tok)</strong>
|
|
</dt>
|
|
<dd>
|
|
get current line number. <br>
|
|
Only available if the input source is a file-like object.
|
|
|
|
<h3>Parameters:</h3>
|
|
<ul>
|
|
<li><code><em>tok</em></code>: a token stream</li>
|
|
</ul>
|
|
|
|
|
|
<h3>Returns:</h3>
|
|
<ol>
|
|
the line number and current column
|
|
</ol>
|
|
|
|
|
|
</dd>
|
|
<dt>
|
|
<a name = "getrest"></a>
|
|
<strong>getrest (tok)</strong>
|
|
</dt>
|
|
<dd>
|
|
get the rest of the stream.
|
|
|
|
<h3>Parameters:</h3>
|
|
<ul>
|
|
<li><code><em>tok</em></code>: a token stream</li>
|
|
</ul>
|
|
|
|
|
|
<h3>Returns:</h3>
|
|
<ol>
|
|
a string
|
|
</ol>
|
|
|
|
|
|
</dd>
|
|
<dt>
|
|
<a name = "get_keywords"></a>
|
|
<strong>get_keywords ()</strong>
|
|
</dt>
|
|
<dd>
|
|
get the Lua keywords as a set-like table.
|
|
So <code>res[“and”]</code> etc would be <code>true</code>.
|
|
|
|
|
|
|
|
<h3>Returns:</h3>
|
|
<ol>
|
|
a table
|
|
</ol>
|
|
|
|
|
|
</dd>
|
|
<dt>
|
|
<a name = "lua"></a>
|
|
<strong>lua (s, filter, options)</strong>
|
|
</dt>
|
|
<dd>
|
|
create a Lua token iterator from a string or file-like object.
|
|
Will return the token type and value.
|
|
|
|
<h3>Parameters:</h3>
|
|
<ul>
|
|
<li><code><em>s</em></code>: the string</li>
|
|
<li><code><em>filter</em></code>: a table of token types to exclude, by default {space=true,comments=true}</li>
|
|
<li><code><em>options</em></code>: a table of options; by default, {number=true,string=true},
|
|
which means convert numbers and strip string quotes.</li>
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
</dd>
|
|
<dt>
|
|
<a name = "cpp"></a>
|
|
<strong>cpp (s, filter, options)</strong>
|
|
</dt>
|
|
<dd>
|
|
create a C/C++ token iterator from a string or file-like object.
|
|
Will return the token type type and value.
|
|
|
|
<h3>Parameters:</h3>
|
|
<ul>
|
|
<li><code><em>s</em></code>: the string</li>
|
|
<li><code><em>filter</em></code>: a table of token types to exclude, by default {space=true,comments=true}</li>
|
|
<li><code><em>options</em></code>: a table of options; by default, {number=true,string=true},
|
|
which means convert numbers and strip string quotes.</li>
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
</dd>
|
|
<dt>
|
|
<a name = "get_separated_list"></a>
|
|
<strong>get_separated_list (tok, endtoken, delim)</strong>
|
|
</dt>
|
|
<dd>
|
|
get a list of parameters separated by a delimiter from a stream.
|
|
|
|
<h3>Parameters:</h3>
|
|
<ul>
|
|
<li><code><em>tok</em></code>: the token stream</li>
|
|
<li><code><em>endtoken</em></code>: end of list (default ‘)’). Can be ‘\n’</li>
|
|
<li><code><em>delim</em></code>: separator (default ‘,’)</li>
|
|
</ul>
|
|
|
|
|
|
<h3>Returns:</h3>
|
|
<ol>
|
|
a list of token lists.
|
|
</ol>
|
|
|
|
|
|
</dd>
|
|
<dt>
|
|
<a name = "skipws"></a>
|
|
<strong>skipws (tok)</strong>
|
|
</dt>
|
|
<dd>
|
|
get the next non-space token from the stream.
|
|
|
|
<h3>Parameters:</h3>
|
|
<ul>
|
|
<li><code><em>tok</em></code>: the token stream.</li>
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
</dd>
|
|
<dt>
|
|
<a name = "expecting"></a>
|
|
<strong>expecting (tok, expected_type, no_skip_ws)</strong>
|
|
</dt>
|
|
<dd>
|
|
get the next token, which must be of the expected type.
|
|
Throws an error if this type does not match!
|
|
|
|
<h3>Parameters:</h3>
|
|
<ul>
|
|
<li><code><em>tok</em></code>: the token stream</li>
|
|
<li><code><em>expected_type</em></code>: the token type</li>
|
|
<li><code><em>no_skip_ws</em></code>: whether we should skip whitespace</li>
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
</dd>
|
|
</dl>
|
|
|
|
|
|
</div> <!-- id="content" -->
|
|
</div> <!-- id="main" -->
|
|
<div id="about">
|
|
<i>generated by <a href="http://github.com/stevedonovan/LDoc">LDoc 1.2</a></i>
|
|
</div> <!-- id="about" -->
|
|
</div> <!-- id="container" -->
|
|
</body>
|
|
</html>
|