From 5f1781a1277508c2b7bec527f722da98d8556e26 Mon Sep 17 00:00:00 2001
From: Mike Pall <mike>
Date: Sun, 21 Apr 2013 01:01:33 +0200
Subject: [PATCH] Compile string concatenations (BC_CAT).

---
 src/Makefile.dep  | 20 +++++-----
 src/jit/dump.lua  |  1 +
 src/lj_asm.c      | 65 ++++++++++++++++++++++++++++++++
 src/lj_asm_arm.h  |  5 +++
 src/lj_asm_mips.h |  5 +++
 src/lj_asm_ppc.h  |  5 +++
 src/lj_asm_x86.h  |  5 +++
 src/lj_buf.c      | 29 +++++++++++++++
 src/lj_buf.h      |  6 +++
 src/lj_ir.c       |  1 +
 src/lj_ir.h       |  9 +++++
 src/lj_ircall.h   | 10 +++--
 src/lj_opt_fold.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++-
 src/lj_record.c   | 35 ++++++++++++++++-
 src/vm_arm.dasc   |  1 +
 src/vm_mips.dasc  |  1 +
 src/vm_ppc.dasc   |  1 +
 src/vm_x86.dasc   |  1 +
 18 files changed, 279 insertions(+), 16 deletions(-)

diff --git a/src/Makefile.dep b/src/Makefile.dep
index 82834811..c501db44 100644
--- a/src/Makefile.dep
+++ b/src/Makefile.dep
@@ -120,9 +120,9 @@ lj_gdbjit.o: lj_gdbjit.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_frame.h lj_bc.h lj_buf.h \
  lj_str.h lj_jit.h lj_ir.h lj_dispatch.h
 lj_ir.o: lj_ir.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
- lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_trace.h \
- lj_dispatch.h lj_bc.h lj_traceerr.h lj_ctype.h lj_cdata.h lj_carith.h \
- lj_vm.h lj_strscan.h lj_lib.h
+ lj_buf.h lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h \
+ lj_trace.h lj_dispatch.h lj_bc.h lj_traceerr.h lj_ctype.h lj_cdata.h \
+ lj_carith.h lj_vm.h lj_strscan.h lj_lib.h
 lj_lex.o: lj_lex.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
  lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_ctype.h lj_cdata.h \
  lualib.h lj_state.h lj_lex.h lj_parse.h lj_char.h lj_strscan.h
@@ -143,9 +143,9 @@ lj_obj.o: lj_obj.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h
 lj_opt_dce.o: lj_opt_dce.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_ir.h lj_jit.h lj_iropt.h
 lj_opt_fold.o: lj_opt_fold.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_iropt.h lj_trace.h lj_dispatch.h \
- lj_bc.h lj_traceerr.h lj_ctype.h lj_gc.h lj_carith.h lj_vm.h \
- lj_strscan.h lj_folddef.h
+ lj_buf.h lj_gc.h lj_str.h lj_tab.h lj_ir.h lj_jit.h lj_iropt.h \
+ lj_trace.h lj_dispatch.h lj_bc.h lj_traceerr.h lj_ctype.h lj_carith.h \
+ lj_vm.h lj_strscan.h lj_folddef.h
 lj_opt_loop.o: lj_opt_loop.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_ir.h lj_jit.h \
  lj_iropt.h lj_trace.h lj_dispatch.h lj_bc.h lj_traceerr.h lj_snap.h \
@@ -165,10 +165,10 @@ lj_parse.o: lj_parse.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_func.h lj_state.h lj_bc.h lj_ctype.h lj_lex.h lj_parse.h lj_vm.h \
  lj_vmevent.h
 lj_record.o: lj_record.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_meta.h lj_frame.h lj_bc.h \
- lj_ctype.h lj_gc.h lj_ff.h lj_ffdef.h lj_ir.h lj_jit.h lj_ircall.h \
- lj_iropt.h lj_trace.h lj_dispatch.h lj_traceerr.h lj_record.h \
- lj_ffrecord.h lj_snap.h lj_vm.h
+ lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_tab.h lj_meta.h \
+ lj_frame.h lj_bc.h lj_ctype.h lj_ff.h lj_ffdef.h lj_ir.h lj_jit.h \
+ lj_ircall.h lj_iropt.h lj_trace.h lj_dispatch.h lj_traceerr.h \
+ lj_record.h lj_ffrecord.h lj_snap.h lj_vm.h
 lj_snap.o: lj_snap.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
  lj_tab.h lj_state.h lj_frame.h lj_bc.h lj_ir.h lj_jit.h lj_iropt.h \
  lj_trace.h lj_dispatch.h lj_traceerr.h lj_snap.h lj_target.h \
diff --git a/src/jit/dump.lua b/src/jit/dump.lua
index 7f930f51..c025a239 100644
--- a/src/jit/dump.lua
+++ b/src/jit/dump.lua
@@ -278,6 +278,7 @@ local litname = {
   ["FLOAD "] = vmdef.irfield,
   ["FREF  "] = vmdef.irfield,
   ["FPMATH"] = vmdef.irfpm,
+  ["BUFHDR"] = { [0] = "RESET", "APPEND" },
 }
 
 local function ctlsub(c)
diff --git a/src/lj_asm.c b/src/lj_asm.c
index 358ace6e..1304c180 100644
--- a/src/lj_asm.c
+++ b/src/lj_asm.c
@@ -1071,6 +1071,70 @@ static void asm_gcstep(ASMState *as, IRIns *ir)
   as->gcsteps = 0x80000000;  /* Prevent implicit GC check further up. */
 }
 
+/* -- Buffer handling ----------------------------------------------------- */
+
+static void asm_bufhdr(ASMState *as, IRIns *ir)
+{
+  if (ra_used(ir)) {
+    Reg sb = ra_dest(as, ir, RSET_GPR);
+    if (!(ir->op2 & IRBUFHDR_APPEND)) {
+      Reg tmp = ra_scratch(as, rset_exclude(RSET_GPR, sb));
+      /* Passing ir isn't strictly correct, but it's an IRT_P32, too. */
+      emit_storeofs(as, ir, tmp, sb, offsetof(SBuf, p));
+      emit_loadofs(as, ir, tmp, sb, offsetof(SBuf, b));
+    }
+#if LJ_TARGET_X86ORX64
+    ra_left(as, sb, ir->op1);
+#else
+    ra_leftov(as, sb, ir->op1);
+#endif
+  }
+}
+
+#if !LJ_TARGET_X86ORX64
+static void asm_tvptr(ASMState *as, Reg dest, IRRef ref);
+#endif
+
+static void asm_bufput(ASMState *as, IRIns *ir)
+{
+  const CCallInfo *ci;
+  IRRef args[2];
+  IRIns *ir2;
+  if (!ra_used(ir)) return;
+  args[0] = ir->op1;  /* SBuf * */
+  args[1] = ir->op2;  /* int, double, GCstr * */
+  ir2 = IR(ir->op2);
+  if (irt_isstr(ir2->t)) {
+    ci = &lj_ir_callinfo[IRCALL_lj_buf_putstr];
+  } else if (LJ_SOFTFP ? irt_type((ir2+1)->t)==IRT_SOFTFP : irt_isnum(ir2->t)) {
+    ci = &lj_ir_callinfo[IRCALL_lj_buf_putnum];
+    args[1] = ASMREF_TMP1;
+  } else {
+    lua_assert(irt_isinteger(ir2->t));
+    ci = &lj_ir_callinfo[IRCALL_lj_buf_putint];
+  }
+  asm_setupresult(as, ir, ci);  /* SBuf * */
+  asm_gencall(as, ci, args);
+  if (args[1] == ASMREF_TMP1) {
+#if LJ_TARGET_X86ORX64
+    emit_rmro(as, XO_LEA, ra_releasetmp(as, ASMREF_TMP1)|REX_64,
+	      RID_ESP, ra_spill(as, IR(ir->op2)));
+#else
+    asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2);
+#endif
+  }
+}
+
+static void asm_bufstr(ASMState *as, IRIns *ir)
+{
+  const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_buf_tostr];
+  IRRef args[1];
+  args[0] = ir->op2;  /* SBuf *sb */
+  as->gcsteps++;
+  asm_setupresult(as, ir, ci);  /* GCstr * */
+  asm_gencall(as, ci, args);
+}
+
 /* -- PHI and loop handling ----------------------------------------------- */
 
 /* Break a PHI cycle by renaming to a free register (evict if needed). */
@@ -1724,6 +1788,7 @@ static void asm_setup_regsp(ASMState *as)
       if (REGARG_NUMGPR < 3 && as->evenspill < 3)
 	as->evenspill = 3;  /* lj_str_new and lj_tab_newkey need 3 args. */
     case IR_TNEW: case IR_TDUP: case IR_CNEW: case IR_CNEWI: case IR_TOSTR:
+    case IR_BUFPUT: case IR_BUFSTR:
       ir->prev = REGSP_HINT(RID_RET);
       if (inloop)
 	as->modset = RSET_SCRATCH;
diff --git a/src/lj_asm_arm.h b/src/lj_asm_arm.h
index 196f797e..57c2dd81 100644
--- a/src/lj_asm_arm.h
+++ b/src/lj_asm_arm.h
@@ -2264,6 +2264,11 @@ static void asm_ir(ASMState *as, IRIns *ir)
   case IR_TDUP: asm_tdup(as, ir); break;
   case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
 
+  /* Buffer operations. */
+  case IR_BUFHDR: asm_bufhdr(as, ir); break;
+  case IR_BUFPUT: asm_bufput(as, ir); break;
+  case IR_BUFSTR: asm_bufstr(as, ir); break;
+
   /* Write barriers. */
   case IR_TBAR: asm_tbar(as, ir); break;
   case IR_OBAR: asm_obar(as, ir); break;
diff --git a/src/lj_asm_mips.h b/src/lj_asm_mips.h
index a1d31d49..55fe10b8 100644
--- a/src/lj_asm_mips.h
+++ b/src/lj_asm_mips.h
@@ -1867,6 +1867,11 @@ static void asm_ir(ASMState *as, IRIns *ir)
   case IR_TDUP: asm_tdup(as, ir); break;
   case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
 
+  /* Buffer operations. */
+  case IR_BUFHDR: asm_bufhdr(as, ir); break;
+  case IR_BUFPUT: asm_bufput(as, ir); break;
+  case IR_BUFSTR: asm_bufstr(as, ir); break;
+
   /* Write barriers. */
   case IR_TBAR: asm_tbar(as, ir); break;
   case IR_OBAR: asm_obar(as, ir); break;
diff --git a/src/lj_asm_ppc.h b/src/lj_asm_ppc.h
index 34bd721f..d0feb43a 100644
--- a/src/lj_asm_ppc.h
+++ b/src/lj_asm_ppc.h
@@ -2065,6 +2065,11 @@ static void asm_ir(ASMState *as, IRIns *ir)
   case IR_TDUP: asm_tdup(as, ir); break;
   case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
 
+  /* Buffer operations. */
+  case IR_BUFHDR: asm_bufhdr(as, ir); break;
+  case IR_BUFPUT: asm_bufput(as, ir); break;
+  case IR_BUFSTR: asm_bufstr(as, ir); break;
+
   /* Write barriers. */
   case IR_TBAR: asm_tbar(as, ir); break;
   case IR_OBAR: asm_obar(as, ir); break;
diff --git a/src/lj_asm_x86.h b/src/lj_asm_x86.h
index d90963ef..1e32b6c9 100644
--- a/src/lj_asm_x86.h
+++ b/src/lj_asm_x86.h
@@ -2707,6 +2707,11 @@ static void asm_ir(ASMState *as, IRIns *ir)
   case IR_TDUP: asm_tdup(as, ir); break;
   case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
 
+  /* Buffer operations. */
+  case IR_BUFHDR: asm_bufhdr(as, ir); break;
+  case IR_BUFPUT: asm_bufput(as, ir); break;
+  case IR_BUFSTR: asm_bufstr(as, ir); break;
+
   /* Write barriers. */
   case IR_TBAR: asm_tbar(as, ir); break;
   case IR_OBAR: asm_obar(as, ir); break;
diff --git a/src/lj_buf.c b/src/lj_buf.c
index c08d23c9..ef48b580 100644
--- a/src/lj_buf.c
+++ b/src/lj_buf.c
@@ -12,6 +12,7 @@
 #include "lj_gc.h"
 #include "lj_err.h"
 #include "lj_buf.h"
+#include "lj_str.h"
 
 LJ_NOINLINE void LJ_FASTCALL lj_buf_grow(SBuf *sb, char *en)
 {
@@ -64,6 +65,34 @@ void lj_buf_putmem(SBuf *sb, const void *q, MSize len)
   setsbufP(sb, p);
 }
 
+#if LJ_HASJIT
+SBuf * LJ_FASTCALL lj_buf_putstr(SBuf *sb, GCstr *s)
+{
+  MSize len = s->len;
+  char *p = lj_buf_more(sb, len);
+  p = lj_buf_wmem(p, strdata(s), len);
+  setsbufP(sb, p);
+  return sb;
+}
+
+SBuf * LJ_FASTCALL lj_buf_putint(SBuf *sb, int32_t k)
+{
+  setsbufP(sb, lj_str_bufint(lj_buf_more(sb, LJ_STR_INTBUF), k));
+  return sb;
+}
+
+SBuf * LJ_FASTCALL lj_buf_putnum(SBuf *sb, cTValue *o)
+{
+  setsbufP(sb, lj_str_bufnum(lj_buf_more(sb, LJ_STR_NUMBUF), o));
+  return sb;
+}
+
+GCstr * LJ_FASTCALL lj_buf_tostr(SBuf *sb)
+{
+  return lj_str_new(sbufL(sb), sbufB(sb), sbuflen(sb));
+}
+#endif
+
 uint32_t LJ_FASTCALL lj_buf_ruleb128(const char **pp)
 {
   const uint8_t *p = (const uint8_t *)*pp;
diff --git a/src/lj_buf.h b/src/lj_buf.h
index 289eb01d..e028a434 100644
--- a/src/lj_buf.h
+++ b/src/lj_buf.h
@@ -26,6 +26,12 @@ LJ_FUNC void LJ_FASTCALL lj_buf_shrink(lua_State *L, SBuf *sb);
 
 LJ_FUNC char *lj_buf_wmem(char *p, const void *q, MSize len);
 LJ_FUNC void lj_buf_putmem(SBuf *sb, const void *q, MSize len);
+#if LJ_HASJIT
+LJ_FUNC SBuf * LJ_FASTCALL lj_buf_putstr(SBuf *sb, GCstr *s);
+LJ_FUNC SBuf * LJ_FASTCALL lj_buf_putint(SBuf *sb, int32_t k);
+LJ_FUNC SBuf * LJ_FASTCALL lj_buf_putnum(SBuf *sb, cTValue *o);
+LJ_FUNC GCstr * LJ_FASTCALL lj_buf_tostr(SBuf *sb);
+#endif
 LJ_FUNC uint32_t LJ_FASTCALL lj_buf_ruleb128(const char **pp);
 LJ_FUNC char * LJ_FASTCALL lj_buf_wuleb128(char *p, uint32_t v);
 
diff --git a/src/lj_ir.c b/src/lj_ir.c
index e1a59105..f1e1959f 100644
--- a/src/lj_ir.c
+++ b/src/lj_ir.c
@@ -15,6 +15,7 @@
 #if LJ_HASJIT
 
 #include "lj_gc.h"
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_ir.h"
diff --git a/src/lj_ir.h b/src/lj_ir.h
index 9d2521c9..0cbd8b55 100644
--- a/src/lj_ir.h
+++ b/src/lj_ir.h
@@ -120,6 +120,11 @@
   _(CNEW,	AW, ref, ref) \
   _(CNEWI,	NW, ref, ref)  /* CSE is ok, not marked as A. */ \
   \
+  /* Buffer operations. */ \
+  _(BUFHDR,	S , ref, lit) \
+  _(BUFPUT,	S , ref, ref) \
+  _(BUFSTR,	A , ref, ref) \
+  \
   /* Barriers. */ \
   _(TBAR,	S , ref, ___) \
   _(OBAR,	S , ref, ref) \
@@ -221,6 +226,10 @@ IRFLDEF(FLENUM)
 #define IRXLOAD_VOLATILE	2	/* Load from volatile data. */
 #define IRXLOAD_UNALIGNED	4	/* Unaligned load. */
 
+/* BUFHDR mode, stored in op2. */
+#define IRBUFHDR_RESET		0	/* Reset buffer. */
+#define IRBUFHDR_APPEND		1	/* Append to buffer. */
+
 /* CONV mode, stored in op2. */
 #define IRCONV_SRCMASK		0x001f	/* Source IRType. */
 #define IRCONV_DSTMASK		0x03e0	/* Dest. IRType (also in ir->t). */
diff --git a/src/lj_ircall.h b/src/lj_ircall.h
index 2c160bdf..3b1cc928 100644
--- a/src/lj_ircall.h
+++ b/src/lj_ircall.h
@@ -105,6 +105,10 @@ typedef struct CCallInfo {
   _(ANY,	lj_strscan_num,		2,  FN, INT, 0) \
   _(ANY,	lj_str_fromint,		2,  FN, STR, CCI_L) \
   _(ANY,	lj_str_fromnum,		2,  FN, STR, CCI_L) \
+  _(ANY,	lj_buf_putstr,		2,  FS, P32, 0) \
+  _(ANY,	lj_buf_putint,		2,  FS, P32, 0) \
+  _(ANY,	lj_buf_putnum,		2,  FS, P32, 0) \
+  _(ANY,	lj_buf_tostr,		1,  FL, STR, 0) \
   _(ANY,	lj_tab_new1,		2,  FS, TAB, CCI_L) \
   _(ANY,	lj_tab_dup,		2,  FS, TAB, CCI_L) \
   _(ANY,	lj_tab_newkey,		3,   S, P32, CCI_L) \
@@ -114,9 +118,9 @@ typedef struct CCallInfo {
   _(ANY,	lj_mem_newgco,		2,  FS, P32, CCI_L) \
   _(ANY,	lj_math_random_step, 1, FS, NUM, CCI_CASTU64|CCI_NOFPRCLOBBER) \
   _(ANY,	lj_vm_modi,		2,  FN, INT, 0) \
-  _(ANY,	sinh,			ARG1_FP,  N, NUM, 0) \
-  _(ANY,	cosh,			ARG1_FP,  N, NUM, 0) \
-  _(ANY,	tanh,			ARG1_FP,  N, NUM, 0) \
+  _(ANY,	sinh,			ARG1_FP,   N, NUM, 0) \
+  _(ANY,	cosh,			ARG1_FP,   N, NUM, 0) \
+  _(ANY,	tanh,			ARG1_FP,   N, NUM, 0) \
   _(ANY,	fputc,			2,  S, INT, 0) \
   _(ANY,	fwrite,			4,  S, INT, 0) \
   _(ANY,	fflush,			1,  S, INT, 0) \
diff --git a/src/lj_opt_fold.c b/src/lj_opt_fold.c
index 75db47df..f35593f3 100644
--- a/src/lj_opt_fold.c
+++ b/src/lj_opt_fold.c
@@ -14,6 +14,7 @@
 
 #if LJ_HASJIT
 
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_ir.h"
@@ -155,13 +156,14 @@ typedef IRRef (LJ_FASTCALL *FoldFunc)(jit_State *J);
 
 /* Barrier to prevent folding across a GC step.
 ** GC steps can only happen at the head of a trace and at LOOP.
-** And the GC is only driven forward if there is at least one allocation.
+** And the GC is only driven forward if there's at least one allocation.
 */
 #define gcstep_barrier(J, ref) \
   ((ref) < J->chain[IR_LOOP] && \
    (J->chain[IR_SNEW] || J->chain[IR_XSNEW] || \
     J->chain[IR_TNEW] || J->chain[IR_TDUP] || \
-    J->chain[IR_CNEW] || J->chain[IR_CNEWI] || J->chain[IR_TOSTR]))
+    J->chain[IR_CNEW] || J->chain[IR_CNEWI] || \
+    J->chain[IR_BUFSTR] || J->chain[IR_TOSTR]))
 
 /* -- Constant folding for FP numbers ------------------------------------- */
 
@@ -515,6 +517,94 @@ LJFOLDF(kfold_strcmp)
   return NEXTFOLD;
 }
 
+/* -- Constant folding and forwarding for buffers ------------------------- */
+
+/* Note: buffer ops are not CSEd until the BUFSTR. It's ok to modify them. */
+
+/* BUFHDR is treated like a store, see below. */
+
+LJFOLD(BUFPUT BUFHDR BUFSTR)
+LJFOLDF(bufput_append)
+{
+  /* New buffer, no other buffer op inbetween and same buffer? */
+  if ((J->flags & JIT_F_OPT_FWD) &&
+      !(fleft->op2 & IRBUFHDR_APPEND) &&
+      fleft->prev == fright->op1 &&
+      fleft->op1 == IR(fright->op1)->op1) {
+    IRRef ref = fins->op1;
+    IR(ref)->op2 = (fleft->op2 | IRBUFHDR_APPEND);  /* Modify BUFHDR. */
+    return ref;
+  }
+  return EMITFOLD;  /* This is a store and always emitted. */
+}
+
+LJFOLD(BUFPUT any any)
+LJFOLDF(bufput_kgc)
+{
+  if (fright->o == IR_KGC) {
+    GCstr *s2 = ir_kstr(fright);
+    MSize len2 = s2->len;
+    if (len2 == 0) {  /* Empty string? */
+      return LEFTFOLD;
+    } else {
+      PHIBARRIER(fleft);
+      if (fleft->o == IR_BUFPUT && IR(fleft->op2)->o == IR_KGC) {
+	/* Join two constant string puts in a row. */
+	GCstr *s1 = ir_kstr(IR(fleft->op2));
+	MSize len1 = s1->len;
+	char *buf = lj_buf_tmp(J->L, len1 + len2);
+	IRRef kref;
+	memcpy(buf, strdata(s1), len1);
+	memcpy(buf+len1, strdata(s2), len2);
+	kref = lj_ir_kstr(J, lj_str_new(J->L, buf, len1 + len2));
+	/* lj_ir_kstr() may realloc the IR and invalidates any IRIns *. */
+	IR(fins->op1)->op2 = kref;  /* Modify previous BUFPUT. */
+	return fins->op1;
+      }
+    }
+  }
+  return EMITFOLD;  /* This is a store and always emitted. */
+}
+
+LJFOLD(BUFSTR any any)
+LJFOLDF(bufstr_kfold_cse)
+{
+  lua_assert(fright->o == IR_BUFHDR || fright->o == IR_BUFPUT);
+  if (fright->o == IR_BUFHDR) {  /* No put operations? */
+    if (!(fright->op2 & IRBUFHDR_APPEND))  /* Empty buffer? */
+      return lj_ir_kstr(J, &J2G(J)->strempty);
+    fins->op2 = fright->prev;  /* Relies on checks in bufput_append. */
+    return CSEFOLD;
+  } else {
+    /* Shortcut for a single put operation. */
+    IRIns *irb = IR(fright->op1);
+    if (irb->o == IR_BUFHDR && !(irb->op2 & IRBUFHDR_APPEND)) {
+      IRRef ref = fright->op2;
+      if (irt_isstr(IR(ref)->t))
+	return ref;
+      lua_assert(irt_isinteger(IR(ref)->t) || irt_isnum(IR(ref)->t));
+      return emitir(IRT(IR_TOSTR, IRT_STR), ref, 0);
+    }
+  }
+  /* Try to CSE the whole chain. */
+  if (LJ_LIKELY(J->flags & JIT_F_OPT_CSE) && !(fleft->op2 & IRBUFHDR_APPEND)) {
+    IRRef ref = J->chain[IR_BUFSTR];
+    while (ref) {
+      IRIns *irs = IR(ref), *ira = fright, *irb = IR(irs->op2);
+      while (ira->o == irb->o && ira->op2 == irb->op2) {
+	if (ira->o == IR_BUFHDR) {
+	  lj_ir_rollback(J, fins->op1);  /* Eliminate the current chain. */
+	  return ref;  /* CSE succeeded. */
+	}
+	ira = IR(ira->op1);
+	irb = IR(irb->op1);
+      }
+      ref = irs->prev;
+    }
+  }
+  return EMITFOLD;  /* No CSE possible. */
+}
+
 /* -- Constant folding of pointer arithmetic ------------------------------ */
 
 LJFOLD(ADD KGC KINT)
@@ -2128,6 +2218,7 @@ LJFOLD(TNEW any any)
 LJFOLD(TDUP any)
 LJFOLD(CNEW any any)
 LJFOLD(XSNEW any any)
+LJFOLD(BUFHDR any any)
 LJFOLDX(lj_ir_emit)
 
 /* ------------------------------------------------------------------------ */
diff --git a/src/lj_record.c b/src/lj_record.c
index 003910a9..bbabd3ce 100644
--- a/src/lj_record.c
+++ b/src/lj_record.c
@@ -11,6 +11,7 @@
 #if LJ_HASJIT
 
 #include "lj_err.h"
+#include "lj_buf.h"
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_meta.h"
@@ -1599,6 +1600,33 @@ static TRef rec_tnew(jit_State *J, uint32_t ah)
   return emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits);
 }
 
+/* -- Concatenation ------------------------------------------------------- */
+
+static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot)
+{
+  TRef *top = &J->base[topslot], tr = *top;
+  lua_assert(baseslot < topslot);
+  if (tref_isnumber_str(tr) && tref_isnumber_str(*(top-1))) {
+    TRef hdr, *trp, *xbase, *base = &J->base[baseslot];
+    /* First convert number consts to string consts to simplify FOLD rules. */
+    for (trp = top; trp >= base && tref_isnumber_str(*trp); trp--)
+      if (tref_isk(*trp) && tref_isnumber(*trp))
+	*trp = emitir(IRT(IR_TOSTR, IRT_STR), *trp, 0);
+    xbase = ++trp;
+    tr = hdr = emitir(IRT(IR_BUFHDR, IRT_P32),
+		      lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET);
+    do {
+      tr = emitir(IRT(IR_BUFPUT, IRT_P32), tr, *trp++);
+    } while (trp <= top);
+    tr = emitir(IRT(IR_BUFSTR, IRT_STR), hdr, tr);
+    J->maxslot = (BCReg)(xbase - J->base);
+    if (xbase == base) return tr;
+  }
+  setintV(&J->errinfo, BC_CAT);
+  lj_trace_err_info(J, LJ_TRERR_NYIBC);  /* __concat metamethod. */
+  return 0;
+}
+
 /* -- Record bytecode ops ------------------------------------------------- */
 
 /* Prepare for comparison. */
@@ -1901,6 +1929,12 @@ void lj_record_ins(jit_State *J)
       rc = rec_mm_arith(J, &ix, MM_pow);
     break;
 
+  /* -- Miscellaneous ops ------------------------------------------------- */
+
+  case BC_CAT:
+    rc = rec_cat(J, rb, rc);
+    break;
+
   /* -- Constant and move ops --------------------------------------------- */
 
   case BC_MOV:
@@ -2082,7 +2116,6 @@ void lj_record_ins(jit_State *J)
     /* fallthrough */
   case BC_ITERN:
   case BC_ISNEXT:
-  case BC_CAT:
   case BC_UCLO:
   case BC_FNEW:
   case BC_TSETM:
diff --git a/src/vm_arm.dasc b/src/vm_arm.dasc
index 1d4b60f4..4579b263 100644
--- a/src/vm_arm.dasc
+++ b/src/vm_arm.dasc
@@ -4344,6 +4344,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  ldr RA, TRACE:RC->mcode
     |   str BASE, [DISPATCH, #DISPATCH_GL(jit_base)]
     |   str L, [DISPATCH, #DISPATCH_GL(jit_L)]
+    |   str L, [DISPATCH, #DISPATCH_GL(tmpbuf.L)]
     |  bx RA
     |.endif
     break;
diff --git a/src/vm_mips.dasc b/src/vm_mips.dasc
index 53000411..c4f01e81 100644
--- a/src/vm_mips.dasc
+++ b/src/vm_mips.dasc
@@ -4051,6 +4051,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |   sw BASE, DISPATCH_GL(jit_base)(DISPATCH)
     |   sw L, DISPATCH_GL(jit_L)(DISPATCH)
     |  lw TMP2, TRACE:TMP2->mcode
+    |   sw L, DISPATCH_GL(tmpbuf.L)(DISPATCH)
     |  jr TMP2
     |.  addiu JGL, DISPATCH, GG_DISP2G+32768
     |.endif
diff --git a/src/vm_ppc.dasc b/src/vm_ppc.dasc
index 514bd231..c85f1f10 100644
--- a/src/vm_ppc.dasc
+++ b/src/vm_ppc.dasc
@@ -4936,6 +4936,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  mtctr TMP2
     |   stw L, DISPATCH_GL(jit_L)(DISPATCH)
     |   addi JGL, DISPATCH, GG_DISP2G+32768
+    |   stw L, DISPATCH_GL(tmpbuf.L)(DISPATCH)
     |  bctr
     |.endif
     break;
diff --git a/src/vm_x86.dasc b/src/vm_x86.dasc
index 3fd897ec..8ed55fd2 100644
--- a/src/vm_x86.dasc
+++ b/src/vm_x86.dasc
@@ -5465,6 +5465,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
     |  mov L:RB, SAVE_L
     |  mov [DISPATCH+DISPATCH_GL(jit_base)], BASE
     |  mov [DISPATCH+DISPATCH_GL(jit_L)], L:RB
+    |  mov [DISPATCH+DISPATCH_GL(tmpbuf.L)], L:RB
     |  // Save additional callee-save registers only used in compiled code.
     |.if X64WIN
     |  mov TMPQ, r12