This file describes the changes on the extension_points branch.


=== Attributes

Attributes are "decorations" of the syntax tree which are ignored by
the type-checker.  An attribute is made of an identifier (an "LIDENT"
or "UIDENT", written id below) and an optional expression (written
expr below).

Attributes on expressions, type expressions, module expressions, module type expressions,
patterns (TODO: class expressions, class type expressions):
 
  ... [@id expr]

The same syntax [@id expr] is also available to add attributes on
constructors and labels in type declarations:

  type t =
    | A [@id1]
    | B [@id2] of int [@id3]

Here, id1 (resp. id2) is attached to the constructor A (resp. B)
and id3 is attached to the int type expression.  Example on records:

 type t =
   {
      x [@id1]: int;
      mutable y [@id2] [@id3]: string [@id4];
   }  


Attributes on items:

  ... [@@id expr]

  Items designate signature and structure items, and also individual
  components of multiple declaration (type declarations, recursive modules,
  class declarations, class type declarations). (TODO: class fields?)

  For instance, consider:

    type t1 = ... [@@id1] [@@id2] and t2 = ... [@@id3] [@@id4]

  Here, the attributes on t1 are id1, id23; the attributes on
  t2 are id3 and id4.

  Note: item attributes are currently not supported on Pstr_eval
  and Pstr_value structure items.

  The [@@id expr] form, when used at the beginning of a signature or
  structure, or after a double semi-colon (;;), defines an attribute
  which stands as a stand-alone signature or structure item (not
  attached to another item).

  Example:

  module type S = sig
    [@@id1]
    type t
    [@@id2]
    ;; [@@id3] [@@id4]
    ;; [@@id5]
    type s
    [@@id6]
  end

  Here, id1, id3, id4, id5 are stand-alone attributes, while
  id2 is attached to the type t and id6 is attached to the type s.

=== Extension nodes

Extension nodes replace valid components in the syntax tree.  They are
normally interpreted and expanded by AST mapper.  The type-checker
fails when it encounters such an extension node.  An extension node is
made of an identifier (an "LIDENT", written id below) and an optional
expression (written expr below).

Two syntaxes exist for extension node:

As expressions, type expressions, module expressions, module type expressions,
patterns (TODO: class expressions, class type expressions):

  [%id expr]

As structure or signature item (TODO: class fields?):
 
  [%%id expr]

As other structure or signature items, attributes can be attached to a
[%%id expr] extension node.


=== Alternative syntax for attributes and extensions on specific kinds of nodes

All expression constructions starting with a keyword, a combination of
keywords or a delimiter supports an alternative syntax for attributes and/or extensions:

  KW[@id expr]...[@id expr] REST
  ---->
  (KW REST)[@id expr]...[@id expr]

  KW%id REST
  ---->
  [%id (KW REST)]

  KW%id[@id expr]...[@id expr] REST
  ---->
  ([%id (KW REST)])[@id expr]...[@id expr]


where KW can stand for:
 (
 (module
 [
 [|
 assert
 begin
 for
 fun
 function
 if
 lazy
 let
 let module
 let open
 match
 new
 object
 try
 while
 {
 {<


For instance:

let[@foo] x = 2 in x + 1   ====   (let x = 2 in x + 1)[@foo]
([@foo] 3 + 4)             ====   (3 + 4)[@foo]
begin[@foo] ... end        ====   (begin ... end)[@foo]
match%foo e with ...       ====   [%foo match e with ...]


=== Representation of attributes in the Parsetree

Attributes as standalone signature/structure items are represented
by a new constructor:

  | Psig_attribute of attribute
  | Pstr_attribute of attribute

Most other attributes are stored in an extra field in their record:

and expression =  {
  ...
  pexp_attributes: attribute list;
  ...
}
and type_declaration = {
  ...
  ptype_attributes: attribute list;
  ...
}

In a previous version, attributes on expressions (and types, patterns,
etc) used to be stored as a new constructor.  The current choice makes
it easier to pattern match on structured AST fragments while ignoring
attributes.

For open/include signature/structure items and exception rebind
structure item, the attributes are stored directly in the constructor
of the item:

  | Pstr_open of Longident.t loc * attribute list



=== Other changes to the parser and Parsetree

--- Introducing Ast_helper module

This module simplifies the creation of AST fragments, without having to
touch the concrete type definitions of Parsetree. Record and sum types
are encapsulated in builder functions, with some optional arguments, e.g.
to represent attributes.

--- Relaxing the syntax for signatures and structures

It is now possible to start a signature or a structure with a ";;" token and to have two successive ";;" tokens.

Rationale:
   It makes it possible to always prefix a "standalone" attribute by ";;" independently
   from its context (this will work at the beginning of the signature/structure and after
   another item finished with ";;").

--- Relaxing the syntax for recursive modules.

Before:
   module X1 : MT1 = M1 and ... and Xn : MTn = Mn

Now:
   module X1 = M1 and ... and Xn = Mn
   (with the usual sugar that Xi = (Mi : MTi) can be written as Xi : MTi = Mi
   which gives the old syntax)

   The type-checker fails when a module expression is not of
   the form (M : MT)
   

Rationale:

1. More uniform representation in the Parsetree.

2. The type-checker can be made more clever in the future to support
   other forms of module expressions (e.g. functions with an explicit
   constraint on its result; or a structure with only type-level
   components).
    
      
--- Turning some tuple or n-ary constructors into records

Before:

  | Pstr_module of string loc * module_expr

After:

  | Pstr_module of module_binding
...
  and module_binding =
    {
     pmb_name: string loc;
     pmb_expr: module_expr;
     pmb_attributes: attribute list;
    }



Rationale:

More self-documented, more robust to future additions (such as
attributes), simplifies some code.


--- Keeping names inside value_description and type_declaration

Before:

  | Psig_type of (string loc * type_declaration) list


After:

  | Psig_type of type_declaration list

....
and type_declaration =
  { ptype_name: string loc;
    ...
  }

Rationale:

More self-documented, simplifies some code.


=== More TODOs

- Adapt pprintast.
- Adapt Camlp4 (both its parser(s) and its internal representation of OCaml ASTs).
- Propagate attributes to the Typedtree (so that they can be retrieved in .cmt/.cmti).
- Consider adding hooks to the type-checker so that custom extension expanders can be registered (a la OCaml Templates).
- Quotations (i.e. string literals with custom delimiters and without any interpretation of special characters in them), and a syntax which combines extension nodes and quotations.
- More cleanups to the Parsetree + documentation.


=== Use cases

From https://github.com/gasche/ocaml-syntax-extension-discussion/wiki/Use-Cases

-- Bisect

  let f x =
  match List.map foo [x; a x; b x] with
  | [y1; y2; y3] -> tata
  | _ -> assert false [@bisect VISIT]

;;[@@bisect IGNORE-BEGIN]
let unused = ()
;;[@@bisect IGNORE-END]

-- OCamldoc

val stats : ('a, 'b) t -> statistics
[@@doc
 "[Hashtbl.stats tbl] returns statistics about the table [tbl]:
  number of buckets, size of the biggest bucket, distribution of
  buckets by size."
]
[@@since "4.00.0"]

;;[@@doc section 6 "Functorial interface"]

module type HashedType =
  sig
    type t
      [@@doc "The type of the hashtable keys."]
    val equal : t -> t -> bool
      [@@doc "The equality predicate used to compare keys."]
  end


-- type-conv, deriving

type t = {
  x : int [@default 42];
  y : int [@default 3] @[sexp_drop_default];
  z : int [@default 3] @[sexp_drop_if z_test];
} [@@sexp]


type r1 = {
  r1_l1 : int;
  r1_l2 : int;
} [@@deriving (Dump, Eq, Show, Typeable, Pickle, Functor)]

-- camlp4 map/fold generators

type variable = string
 and term =
  | Var of variable
  | Lam of variable * term
  | App of term * term


class map = [%generate_map term]
or:
[%%generate_map map term]


-- ocaml-rpc

type t = { foo [@rpc "type"]: int; bar [@rpc "let"]: int }
[@@ rpc]

or:

type t = { foo: int; bar: int }
[@@ rpc ("foo" > "type"), ("bar" > "let")]



-- pa_monad

begin%monad
  a <-- [1; 2; 3];
  b <-- [3; 4; 5];
  return (a + b)
end

-- pa_lwt

let%lwt x = start_thread foo
and y = start_other_thread foo in
try%lwt
  let%for_lwt (x, y) = waiting_threads in
  compute blah
with Killed -> bar

-- Bolt

let funct n =
  (%log "funct(%d)" n LEVEL DEBUG);
  for i = 1 to n do
    print_endline "..."
  done


-- pre-polyrecord

let r = {%polyrec x = 1; y = ref None }
let () = (%polyrec r.y <- Some 2)

-- orakuda

function%regexp
  | "$/^[0-9]+$/" as v -> `Int (int_of_string v#_0)
  | "$/^[a-z][A-Za-z0-9_]*$" as v -> `Variable v#_0
  | _ -> failwith "parse error"

-- bitstring

let bits = Bitstring.bitstring_of_file "/bin/ls" in
match%bitstring bits with
| [ 0x7f, 8; "ELF", 24, string;  (* ELF magic number *)
    e_ident, Mul(12,8), bitstring;    (* ELF identifier *)
    e_type, 16, littleendian;    (* object file type *)
    e_machine, 16, littleendian  (* architecture *)
  ] ->
  printf "This is an ELF binary, type %d, arch %d\n"
    e_type e_machine

-- sedlex

let rec token buf =
  let%regexp ('a'..'z'|'A'..'Z') = letter in
  match%sedlex buf with
  | number -> Printf.printf "Number %s\n" (Sedlexing.Latin1.lexeme buf); token buf
  | letter, Star ('A'..'Z' | 'a'..'z' | digit) -> Printf.printf "Ident %s\n" (Sedlexing.Latin1.lexeme buf); token buf
  | Plus xml_blank -> token buf
  | Plus (Chars "+*-/") -> Printf.printf "Op %s\n" (Sedlexing.Latin1.lexeme buf); token buf
  | Range(128,255) -> print_endline "Non ASCII"
  | eof -> print_endline "EOF"
  | _ -> failwith "Unexpected character"


-- cppo

[%%ifdef DEBUG]
[%%define debug(s) = Printf.eprintf "[%S %i] %s\n%!" __FILE__ __LINE__ s]
[%%else]
[%%define debug(s) = ()]
[%%endif]

debug("test")


-- PG'OCaml

let fetch_users dbh =
  (%pgsql dbh "select id, name from users")


-- Macaque

let names view = (%view {name = t.name}, t <- !view)