Support marshalling of bigarrays with dimensions that don't fit in 4 bytes (#8791)

Use a variable-length encoding (suggested by @stedolan) for dimensions that supports dimensions up to 2^64-1 each.

Change the marshalling identifier for bigarrays:_bigarray ~> _bigarr02
The identifier change reflects a change in the bigarray marshalling format.

Changes

@@ -140,6 +140,10 @@ OCaml 4.11
   (Mark Shinwell, David Allsopp, Vincent Laviron, Xavier Leroy,
   Geoff Reedy, original bug report by Arlen Cox)
 
+- #8791: use a variable-length encoding when marshalling bigarray dimensions,
+  avoiding overflow.
+  (Jeremy Yallop, Stephen Dolan, review by Xavier Leroy)
+
 ### Code generation and optimizations:
 
 - #8637, #8805, #9247, #9296: Record debug info for each allocation.

runtime/bigarray.c

@@ -67,7 +67,7 @@ CAMLexport uintnat caml_ba_byte_size(struct caml_ba_array * b)
 /* Operation table for bigarrays */
 
 CAMLexport struct custom_operations caml_ba_ops = {
-  "_bigarray",
+  "_bigarr02",
   caml_ba_finalize,
   caml_ba_compare,
   caml_ba_hash,
@@ -374,11 +374,15 @@ CAMLexport void caml_ba_serialize(value v,
   /* Serialize header information */
   caml_serialize_int_4(b->num_dims);
   caml_serialize_int_4(b->flags & (CAML_BA_KIND_MASK | CAML_BA_LAYOUT_MASK));
-  /* On a 64-bit machine, if any of the dimensions is >= 2^32,
-     the size of the marshaled data will be >= 2^32 and
-     extern_value() will fail.  So, it is safe to write the dimensions
-     as 32-bit unsigned integers. */
-  for (i = 0; i < b->num_dims; i++) caml_serialize_int_4(b->dim[i]);
+  for (i = 0; i < b->num_dims; i++) {
+    intnat len = b->dim[i];
+    if (len < 0xffff) {
+      caml_serialize_int_2(len);
+    } else {
+      caml_serialize_int_2(0xffff);
+      caml_serialize_int_8(len);
+    }
+  }
   /* Compute total number of elements */
   num_elts = 1;
   for (i = 0; i < b->num_dims; i++) num_elts = num_elts * b->dim[i];
@@ -446,7 +450,11 @@ CAMLexport uintnat caml_ba_deserialize(void * dst)
     caml_deserialize_error("input_value: wrong number of bigarray dimensions");
   b->flags = caml_deserialize_uint_4() | CAML_BA_MANAGED;
   b->proxy = NULL;
-  for (i = 0; i < b->num_dims; i++) b->dim[i] = caml_deserialize_uint_4();
+  for (i = 0; i < b->num_dims; i++) {
+    intnat len = caml_deserialize_uint_2();
+    if (len == 0xffff) len = caml_deserialize_uint_8();
+    b->dim[i] = len;
+  }
   /* Compute total number of elements.  Watch out for overflows (MPR#7765). */
   num_elts = 1;
   for (i = 0; i < b->num_dims; i++) {

testsuite/tests/lib-marshal/marshal_bigarray.ml

@@ -0,0 +1,14 @@
+(* TEST *)
+
+let () =
+  let small = 0xfffe and large = 0xffff in
+  let marshalled dim =
+    let ba = Bigarray.(Array1.create int8_unsigned c_layout dim) in
+    Marshal.to_string ba []
+  in
+  (* Bigarray dimension marshalling scheme: use an extra 8 bytes
+     to marshal dimensions >=0xffff to avoid overflow *)
+  assert
+    (((String.length (marshalled large) - String.length (marshalled small))
+      - (large - small))
+     = 8)