patch 8.2.0803: libvterm code lags behind the upstream version

Problem:    Libvterm code lags behind the upstream version.
Solution:   Include revisions 764 - 767

Filelist

@@ -301,6 +301,7 @@ SRC_ALL =	\
 		src/libvterm/LICENSE \
 		src/libvterm/Makefile \
 		src/libvterm/README \
+		src/libvterm/CONTRIBUTING \
 		src/libvterm/tbl2inc_c.pl \
 		src/libvterm/vterm.pc.in \
 		src/libvterm/doc/URLs \

runtime/doc/popup.txt

@@ -150,7 +150,7 @@ different:						*E863*
 - When the job ends, the popup window closes.
 - The popup window can be closed with `popup_close()`, the terminal buffer
   then becomes hidden.
-- It is not possible to enter Terminal-Normal mode.
+- It is not possible to open a second popup window with a terminal. *E861*
 - The default Pmenu color is only used for the border and padding.  To change
   the color of the terminal itself set the Terminal highlight group before
   creating the terminal.  Setting 'wincolor' later can work but requires the

src/Makefile

@@ -3562,7 +3562,7 @@ objects/vterm_screen.o: libvterm/src/screen.c $(TERM_DEPS)
 objects/vterm_state.o: libvterm/src/state.c $(TERM_DEPS)
 	$(CCCTERM) -o $@ libvterm/src/state.c
 
-objects/vterm_unicode.o: libvterm/src/unicode.c $(TERM_DEPS)
+objects/vterm_unicode.o: libvterm/src/unicode.c $(TERM_DEPS) libvterm/src/fullwidth.inc
 	$(CCCTERM) -o $@ libvterm/src/unicode.c
 
 objects/vterm_vterm.o: libvterm/src/vterm.c $(TERM_DEPS)

src/ex_docmd.c

@@ -2918,8 +2918,12 @@ parse_cmd_address(exarg_T *eap, char **errormsg, int silent)
 	{
 	    case ADDR_LINES:
 	    case ADDR_OTHER:
-		// default is current line number
-		eap->line2 = curwin->w_cursor.lnum;
+		// Default is the cursor line number.  Avoid using an invalid
+		// line number though.
+		if (curwin->w_cursor.lnum > curbuf->b_ml.ml_line_count)
+		    eap->line2 = curbuf->b_ml.ml_line_count;
+		else
+		    eap->line2 = curwin->w_cursor.lnum;
 		break;
 	    case ADDR_WINDOWS:
 		eap->line2 = CURRENT_WIN_NR;

src/json.c

@@ -20,6 +20,8 @@
 
 static int json_encode_item(garray_T *gap, typval_T *val, int copyID, int options);
 
+static char e_json_error[] = N_("E491: json decode error at '%s'");
+
 /*
  * Encode "val" into a JSON format string.
  * The result is added to "gap"
@@ -740,7 +742,7 @@ json_decode_item(js_read_T *reader, typval_T *res, int options)
 			retval = json_decode_string(reader, cur_item, *p);
 		    else
 		    {
-			emsg(_(e_invarg));
+			semsg(_(e_json_error), p);
 			retval = FAIL;
 		    }
 		    break;
@@ -748,7 +750,7 @@ json_decode_item(js_read_T *reader, typval_T *res, int options)
 		case ',': // comma: empty item
 		    if ((options & JSON_JS) == 0)
 		    {
-			emsg(_(e_invarg));
+			semsg(_(e_json_error), p);
 			retval = FAIL;
 			break;
 		    }
@@ -778,7 +780,7 @@ json_decode_item(js_read_T *reader, typval_T *res, int options)
 			    }
 			    if (!VIM_ISDIGIT(*sp))
 			    {
-				emsg(_(e_invarg));
+				semsg(_(e_json_error), p);
 				retval = FAIL;
 				break;
 			    }
@@ -809,7 +811,7 @@ json_decode_item(js_read_T *reader, typval_T *res, int options)
 				    &nr, NULL, 0, TRUE);
 			    if (len == 0)
 			    {
-				emsg(_(e_invarg));
+				semsg(_(e_json_error), p);
 				retval = FAIL;
 				goto theend;
 			    }
@@ -962,7 +964,7 @@ item_end:
 			retval = MAYBE;
 		    else
 		    {
-			emsg(_(e_invarg));
+			semsg(_(e_json_error), p);
 			retval = FAIL;
 		    }
 		    goto theend;
@@ -980,7 +982,7 @@ item_end:
 			retval = MAYBE;
 		    else
 		    {
-			emsg(_(e_invarg));
+			semsg(_(e_json_error), p);
 			retval = FAIL;
 		    }
 		    goto theend;
@@ -1036,7 +1038,7 @@ item_end:
 			retval = MAYBE;
 		    else
 		    {
-			emsg(_(e_invarg));
+			semsg(_(e_json_error), p);
 			retval = FAIL;
 		    }
 		    goto theend;
@@ -1055,7 +1057,7 @@ item_end:
 	res->v_type = VAR_SPECIAL;
 	res->vval.v_number = VVAL_NONE;
     }
-    emsg(_(e_invarg));
+    semsg(_(e_json_error), p);
 
 theend:
     ga_clear(&stack);
@@ -1079,7 +1081,7 @@ json_decode_all(js_read_T *reader, typval_T *res, int options)
     if (ret != OK)
     {
 	if (ret == MAYBE)
-	    emsg(_(e_invarg));
+	    semsg(_(e_json_error), reader->js_buf);
 	return FAIL;
     }
     json_skip_white(reader);

src/libvterm/CONTRIBUTING

@@ -0,0 +1,22 @@
+How to Contribute
+-----------------
+
+The main resources for this library are:
+
+  Launchpad
+    https://launchpad.net/libvterm
+
+  Freenode:
+    ##tty or #tickit on irc.freenode.net
+
+  Email:
+    Paul "LeoNerd" Evans <leonerd@leonerd.org.uk>
+
+
+Bug reports and feature requests can be sent to any of the above resources.
+
+New features, bug patches, etc.. should in the first instance be discussed via
+any of the resources listed above, before starting work on the actual code.
+There may be future plans or development already in-progress that could be
+affected so it is better to discuss the ideas first before starting work
+actually writing any code.

src/libvterm/Makefile

@@ -37,13 +37,13 @@ INCFILES=$(TBLFILES:.tbl=.inc)
 HFILES_INT=$(sort $(wildcard src/*.h)) $(HFILES)
 
 VERSION_MAJOR=0
-VERSION_MINOR=0
+VERSION_MINOR=1
 
 VERSION_CURRENT=0
 VERSION_REVISION=0
 VERSION_AGE=0
 
-VERSION=0
+VERSION=$(VERSION_MAJOR).$(VERSION_MINOR)
 
 PREFIX=/usr/local
 BINDIR=$(PREFIX)/bin
@@ -110,13 +110,11 @@ install-bin: $(BINFILES)
 
 # DIST CUT
 
-VERSION=$(VERSION_MAJOR).$(VERSION_MINOR)
-
 DISTDIR=libvterm-$(VERSION)
 
 distdir: $(INCFILES)
 	mkdir __distdir
-	cp LICENSE __distdir
+	cp LICENSE CONTRIBUTING __distdir
 	mkdir __distdir/src
 	cp src/*.c src/*.h src/*.inc __distdir/src
 	mkdir __distdir/src/encoding

src/libvterm/doc/seqs.txt

@@ -167,8 +167,10 @@ between states.
 123   SGR 1            = Bold on
       SGR 3            = Italic on
 123   SGR 4            = Underline single
+      SGR 4:x          = Underline style
 123   SGR 5            = Blink on
 123   SGR 7            = Reverse on
+      SGR 8            = Conceal on
       SGR 9            = Strikethrough on
       SGR 10-19        = Select font
       SGR 21           = Underline double
@@ -177,6 +179,7 @@ between states.
  23   SGR 24           = Underline off
  23   SGR 25           = Blink off
  23   SGR 27           = Reverse off
+      SGR 28           = Conceal off
       SGR 29           = Strikethrough off
       SGR 30-37        = Foreground ANSI
       SGR 38           = Foreground alternative palette

src/libvterm/find-wide-chars.pl

@@ -3,16 +3,12 @@
 use strict;
 use warnings;
 
-use Unicode::UCD qw( charprop );
-
 STDOUT->autoflush(1);
 
 sub iswide
 {
    my ( $cp ) = @_;
-
-   my $width = charprop( $cp, "East_Asian_Width" ) or return;
-   return $width eq "Wide" || $width eq "Fullwidth";
+   return chr($cp) =~ m/\p{East_Asian_Width=Wide}|\p{East_Asian_Width=Fullwidth}/;
 }
 
 my ( $start, $end );

src/libvterm/include/vterm.h

@@ -19,6 +19,12 @@ extern "C" {
 typedef unsigned char		uint8_t;
 typedef unsigned int		uint32_t;
 
+#define VTERM_VERSION_MAJOR 0
+#define VTERM_VERSION_MINOR 1
+
+#define VTERM_CHECK_VERSION \
+        vterm_check_version(VTERM_VERSION_MAJOR, VTERM_VERSION_MINOR)
+
 typedef struct VTerm VTerm;
 typedef struct VTermState VTermState;
 typedef struct VTermScreen VTermScreen;
@@ -101,10 +107,17 @@ typedef enum {
   VTERM_N_VALUETYPES
 } VTermValueType;
 
+typedef struct {
+  const char *str;
+  size_t      len : 30;
+  unsigned int  initial : 1;
+  unsigned int  final : 1;
+} VTermStringFragment;
+
 typedef union {
   int boolean;
   int number;
-  char *string;
+  VTermStringFragment string;
   VTermColor color;
 } VTermValue;
 
@@ -115,6 +128,7 @@ typedef enum {
   VTERM_ATTR_ITALIC,     // bool:   3, 23
   VTERM_ATTR_BLINK,      // bool:   5, 25
   VTERM_ATTR_REVERSE,    // bool:   7, 27
+  VTERM_ATTR_CONCEAL,    // bool:   8, 28
   VTERM_ATTR_STRIKE,     // bool:   9, 29
   VTERM_ATTR_FONT,       // number: 10-19
   VTERM_ATTR_FOREGROUND, // color:  30-39 90-97
@@ -164,8 +178,9 @@ typedef struct {
 } VTermGlyphInfo;
 
 typedef struct {
-  unsigned int    doublewidth:1;     // DECDWL or DECDHL line
-  unsigned int    doubleheight:2;    // DECDHL line (1=top 2=bottom)
+  unsigned int    doublewidth:1;     /* DECDWL or DECDHL line */
+  unsigned int    doubleheight:2;    /* DECDHL line (1=top 2=bottom) */
+  unsigned int    continuation:1;    /* Line is a flow continuation of the previous */
 } VTermLineInfo;
 
 typedef struct {
@@ -175,6 +190,8 @@ typedef struct {
   void  (*free)(void *ptr, void *allocdata);
 } VTermAllocatorFunctions;
 
+void vterm_check_version(int major, int minor);
+
 // Allocate and initialize a new terminal with default allocators.
 VTerm *vterm_new(int rows, int cols);
 
@@ -248,8 +265,8 @@ typedef struct {
   int (*control)(unsigned char control, void *user);
   int (*escape)(const char *bytes, size_t len, void *user);
   int (*csi)(const char *leader, const long args[], int argcount, const char *intermed, char command, void *user);
-  int (*osc)(const char *command, size_t cmdlen, void *user);
-  int (*dcs)(const char *command, size_t cmdlen, void *user);
+  int (*osc)(int command, VTermStringFragment frag, void *user);
+  int (*dcs)(const char *command, size_t commandlen, VTermStringFragment frag, void *user);
   int (*resize)(int rows, int cols, void *user);
 } VTermParserCallbacks;
 
@@ -260,6 +277,15 @@ void *vterm_parser_get_cbdata(VTerm *vt);
 // State layer
 // -----------
 
+/* Copies of VTermState fields that the 'resize' callback might have reason to
+ * edit. 'resize' callback gets total control of these fields and may
+ * free-and-reallocate them if required. They will be copied back from the
+ * struct after the callback has returned.
+ */
+typedef struct {
+  VTermPos pos;                /* current cursor position */
+} VTermStateFields;
+
 typedef struct {
   int (*putglyph)(VTermGlyphInfo *info, VTermPos pos, void *user);
   int (*movecursor)(VTermPos pos, VTermPos oldpos, int visible, void *user);
@@ -272,7 +298,7 @@ typedef struct {
   // was accepted, 0 otherwise.
   int (*settermprop)(VTermProp prop, VTermValue *val, void *user);
   int (*bell)(void *user);
-  int (*resize)(int rows, int cols, VTermPos *delta, void *user);
+  int (*resize)(int rows, int cols, VTermStateFields *fields, void *user);
   int (*setlineinfo)(int row, const VTermLineInfo *newinfo, const VTermLineInfo *oldinfo, void *user);
 } VTermStateCallbacks;
 
@@ -289,13 +315,19 @@ typedef struct {
   // useful to add protocol?
 } VTermMouseState;
 
+typedef struct {
+  int (*control)(unsigned char control, void *user);
+  int (*csi)(const char *leader, const long args[], int argcount, const char *intermed, char command, void *user);
+  int (*osc)(int command, VTermStringFragment frag, void *user);
+  int (*dcs)(const char *command, size_t commandlen, VTermStringFragment frag, void *user);
+} VTermStateFallbacks;
+
 VTermState *vterm_obtain_state(VTerm *vt);
 
 void  vterm_state_set_callbacks(VTermState *state, const VTermStateCallbacks *callbacks, void *user);
 void *vterm_state_get_cbdata(VTermState *state);
 
-// Only invokes control, csi, osc, dcs
-void  vterm_state_set_unrecognised_fallbacks(VTermState *state, const VTermParserCallbacks *fallbacks, void *user);
+void  vterm_state_set_unrecognised_fallbacks(VTermState *state, const VTermStateFallbacks *fallbacks, void *user);
 void *vterm_state_get_unrecognised_fbdata(VTermState *state);
 
 // Initialize the state.
@@ -324,12 +356,20 @@ typedef struct {
     unsigned int italic    : 1;
     unsigned int blink     : 1;
     unsigned int reverse   : 1;
+    unsigned int conceal   : 1;
     unsigned int strike    : 1;
     unsigned int font      : 4; // 0 to 9
     unsigned int dwl       : 1; // On a DECDWL or DECDHL line
     unsigned int dhl       : 2; // On a DECDHL line (1=top 2=bottom)
 } VTermScreenCellAttrs;
 
+enum {
+  VTERM_UNDERLINE_OFF,
+  VTERM_UNDERLINE_SINGLE,
+  VTERM_UNDERLINE_DOUBLE,
+  VTERM_UNDERLINE_CURLY,
+};
+
 typedef struct {
 #define VTERM_MAX_CHARS_PER_CELL 6
   uint32_t chars[VTERM_MAX_CHARS_PER_CELL];
@@ -363,8 +403,7 @@ VTermScreen *vterm_obtain_screen(VTerm *vt);
 void  vterm_screen_set_callbacks(VTermScreen *screen, const VTermScreenCallbacks *callbacks, void *user);
 void *vterm_screen_get_cbdata(VTermScreen *screen);
 
-// Only invokes control, csi, osc, dcs
-void  vterm_screen_set_unrecognised_fallbacks(VTermScreen *screen, const VTermParserCallbacks *fallbacks, void *user);
+void  vterm_screen_set_unrecognised_fallbacks(VTermScreen *screen, const VTermStateFallbacks *fallbacks, void *user);
 void *vterm_screen_get_unrecognised_fbdata(VTermScreen *screen);
 
 // Enable support for using the alternate screen if "altscreen" is non-zero.
@@ -406,8 +445,9 @@ typedef enum {
   VTERM_ATTR_FONT_MASK       = 1 << 6,
   VTERM_ATTR_FOREGROUND_MASK = 1 << 7,
   VTERM_ATTR_BACKGROUND_MASK = 1 << 8,
+  VTERM_ATTR_CONCEAL_MASK    = 1 << 9,
 
-  VTERM_ALL_ATTRS_MASK = (1 << 9) - 1
+  VTERM_ALL_ATTRS_MASK = (1 << 10) - 1
 } VTermAttrMask;
 
 int vterm_screen_get_attrs_extent(const VTermScreen *screen, VTermRect *extent, VTermPos pos, VTermAttrMask attrs);

src/libvterm/src/fullwidth.inc

@@ -40,14 +40,13 @@
   { 0x3000, 0x303e },
   { 0x3041, 0x3096 },
   { 0x3099, 0x30ff },
-  { 0x3105, 0x312d },
+  { 0x3105, 0x312f },
   { 0x3131, 0x318e },
   { 0x3190, 0x31ba },
   { 0x31c0, 0x31e3 },
   { 0x31f0, 0x321e },
   { 0x3220, 0x3247 },
-  { 0x3250, 0x32fe },
-  { 0x3300, 0x4dbf },
+  { 0x3250, 0x4dbf },
   { 0x4e00, 0xa48c },
   { 0xa490, 0xa4c6 },
   { 0xa960, 0xa97c },
@@ -59,10 +58,13 @@
   { 0xfe68, 0xfe6b },
   { 0xff01, 0xff60 },
   { 0xffe0, 0xffe6 },
-  { 0x16fe0, 0x16fe0 },
-  { 0x17000, 0x187ec },
+  { 0x16fe0, 0x16fe3 },
+  { 0x17000, 0x187f7 },
   { 0x18800, 0x18af2 },
-  { 0x1b000, 0x1b001 },
+  { 0x1b000, 0x1b11e },
+  { 0x1b150, 0x1b152 },
+  { 0x1b164, 0x1b167 },
+  { 0x1b170, 0x1b2fb },
   { 0x1f004, 0x1f004 },
   { 0x1f0cf, 0x1f0cf },
   { 0x1f18e, 0x1f18e },
@@ -71,6 +73,7 @@
   { 0x1f210, 0x1f23b },
   { 0x1f240, 0x1f248 },
   { 0x1f250, 0x1f251 },
+  { 0x1f260, 0x1f265 },
   { 0x1f300, 0x1f320 },
   { 0x1f32d, 0x1f335 },
   { 0x1f337, 0x1f37c },
@@ -92,13 +95,17 @@
   { 0x1f680, 0x1f6c5 },
   { 0x1f6cc, 0x1f6cc },
   { 0x1f6d0, 0x1f6d2 },
+  { 0x1f6d5, 0x1f6d5 },
   { 0x1f6eb, 0x1f6ec },
-  { 0x1f6f4, 0x1f6f6 },
-  { 0x1f910, 0x1f91e },
-  { 0x1f920, 0x1f927 },
-  { 0x1f930, 0x1f930 },
-  { 0x1f933, 0x1f93e },
-  { 0x1f940, 0x1f94b },
-  { 0x1f950, 0x1f95e },
-  { 0x1f980, 0x1f991 },
-  { 0x1f9c0, 0x1f9c0 },
+  { 0x1f6f4, 0x1f6fa },
+  { 0x1f7e0, 0x1f7eb },
+  { 0x1f90d, 0x1f971 },
+  { 0x1f973, 0x1f976 },
+  { 0x1f97a, 0x1f9a2 },
+  { 0x1f9a5, 0x1f9aa },
+  { 0x1f9ae, 0x1f9ca },
+  { 0x1f9cd, 0x1f9ff },
+  { 0x1fa70, 0x1fa73 },
+  { 0x1fa78, 0x1fa7a },
+  { 0x1fa80, 0x1fa82 },
+  { 0x1fa90, 0x1fa95 },

src/libvterm/src/parser.c

@@ -23,10 +23,10 @@ static void do_csi(VTerm *vt, char command)
 {
 #ifdef DEBUG_PARSER
   printf("Parsed CSI args as:\n", arglen, args);
-  printf(" leader: %s\n", vt->parser.csi_leader);
-  for(int argi = 0; argi < vt->parser.csi_argi; argi++) {
-    printf(" %lu", CSI_ARG(vt->parser.csi_args[argi]));
-    if(!CSI_ARG_HAS_MORE(vt->parser.csi_args[argi]))
+  printf(" leader: %s\n", vt->parser.v.csi.leader);
+  for(int argi = 0; argi < vt->parser.v.csi.argi; argi++) {
+    printf(" %lu", CSI_ARG(vt->parser.v.csi.args[argi]));
+    if(!CSI_ARG_HAS_MORE(vt->parser.v.csi.args[argi]))
       printf("\n");
   printf(" intermed: %s\n", vt->parser.intermed);
   }
@@ -34,9 +34,9 @@ static void do_csi(VTerm *vt, char command)
 
   if(vt->parser.callbacks && vt->parser.callbacks->csi)
     if((*vt->parser.callbacks->csi)(
-          vt->parser.csi_leaderlen ? vt->parser.csi_leader : NULL,
-          vt->parser.csi_args,
-          vt->parser.csi_argi,
+          vt->parser.v.csi.leaderlen ? vt->parser.v.csi.leader : NULL,
+          vt->parser.v.csi.args,
+          vt->parser.v.csi.argi,
           vt->parser.intermedlen ? vt->parser.intermed : NULL,
           command,
           vt->parser.cbdata))
@@ -61,65 +61,36 @@ static void do_escape(VTerm *vt, char command)
   DEBUG_LOG1("libvterm: Unhandled escape ESC 0x%02x\n", command);
 }
 
-static void append_strbuffer(VTerm *vt, const char *str, size_t len)
+static void string_fragment(VTerm *vt, const char *str, size_t len, int final)
 {
-  if(len > vt->parser.strbuffer_len - vt->parser.strbuffer_cur) {
-    len = vt->parser.strbuffer_len - vt->parser.strbuffer_cur;
-    DEBUG_LOG1("Truncating strbuffer preserve to %zu bytes\n", len);
+  VTermStringFragment frag;
+
+  frag.str = str;
+  frag.len = len;
+  frag.initial = vt->parser.string_initial;
+  frag.final = final;
+
+  switch(vt->parser.state) {
+    case OSC:
+      if(vt->parser.callbacks && vt->parser.callbacks->osc)
+        (*vt->parser.callbacks->osc)(vt->parser.v.osc.command, frag, vt->parser.cbdata);
+      break;
+
+    case DCS:
+      if(len && vt->parser.callbacks && vt->parser.callbacks->dcs)
+        (*vt->parser.callbacks->dcs)(vt->parser.v.dcs.command, vt->parser.v.dcs.commandlen, frag, vt->parser.cbdata);
+      break;
+
+    case NORMAL:
+    case CSI_LEADER:
+    case CSI_ARGS:
+    case CSI_INTERMED:
+    case OSC_COMMAND:
+    case DCS_COMMAND:
+      break;
   }
 
-  if(len > 0) {
-    strncpy(vt->parser.strbuffer + vt->parser.strbuffer_cur, str, len);
-    vt->parser.strbuffer_cur += len;
-  }
-}
-
-static void start_string(VTerm *vt, VTermParserStringType type)
-{
-  vt->parser.stringtype = type;
-
-  vt->parser.strbuffer_cur = 0;
-}
-
-static void more_string(VTerm *vt, const char *str, size_t len)
-{
-  append_strbuffer(vt, str, len);
-}
-
-static void done_string(VTerm *vt, const char *str, size_t len)
-{
-  if(vt->parser.strbuffer_cur) {
-    if(str)
-      append_strbuffer(vt, str, len);
-
-    str = vt->parser.strbuffer;
-    len = vt->parser.strbuffer_cur;
-  }
-  else if(!str) {
-    DEBUG_LOG("parser.c: TODO: No strbuffer _and_ no final fragment???\n");
-    len = 0;
-  }
-
-  switch(vt->parser.stringtype) {
-  case VTERM_PARSER_OSC:
-    if(vt->parser.callbacks && vt->parser.callbacks->osc)
-      if((*vt->parser.callbacks->osc)(str, len, vt->parser.cbdata))
-        return;
-
-    DEBUG_LOG2("libvterm: Unhandled OSC %.*s\n", (int)len, str);
-    return;
-
-  case VTERM_PARSER_DCS:
-    if(vt->parser.callbacks && vt->parser.callbacks->dcs)
-      if((*vt->parser.callbacks->dcs)(str, len, vt->parser.cbdata))
-        return;
-
-    DEBUG_LOG2("libvterm: Unhandled DCS %.*s\n", (int)len, str);
-    return;
-
-  case VTERM_N_PARSER_TYPES:
-    return;
-  }
+  vt->parser.string_initial = FALSE;
 }
 
 size_t vterm_input_write(VTerm *vt, const char *bytes, size_t len)
@@ -135,43 +106,47 @@ size_t vterm_input_write(VTerm *vt, const char *bytes, size_t len)
   case CSI_LEADER:
   case CSI_ARGS:
   case CSI_INTERMED:
-  case ESC:
+  case OSC_COMMAND:
+  case DCS_COMMAND:
     string_start = NULL;
     break;
-  case STRING:
-  case ESC_IN_STRING:
+  case OSC:
+  case DCS:
     string_start = bytes;
     break;
   }
 
-#define ENTER_STRING_STATE()   do { vt->parser.state = STRING; string_start = bytes + pos + 1; } while(0)
 #define ENTER_STATE(st)        do { vt->parser.state = st; string_start = NULL; } while(0)
 #define ENTER_NORMAL_STATE()   ENTER_STATE(NORMAL)
 
+#define IS_STRING_STATE()      (vt->parser.state >= OSC_COMMAND)
+
   for( ; pos < len; pos++) {
     unsigned char c = bytes[pos];
+    int c1_allowed = !vt->mode.utf8;
+    size_t string_len;
 
     if(c == 0x00 || c == 0x7f) { // NUL, DEL
-      if(vt->parser.state >= STRING) {
-        more_string(vt, string_start, bytes + pos - string_start);
+      if(IS_STRING_STATE()) {
+        string_fragment(vt, string_start, bytes + pos - string_start, FALSE);
         string_start = bytes + pos + 1;
       }
       continue;
     }
     if(c == 0x18 || c == 0x1a) { // CAN, SUB
+      vt->parser.in_esc = FALSE;
       ENTER_NORMAL_STATE();
       continue;
     }
     else if(c == 0x1b) { // ESC
       vt->parser.intermedlen = 0;
-      if(vt->parser.state == STRING)
-        vt->parser.state = ESC_IN_STRING;
-      else
-        ENTER_STATE(ESC);
+      if(!IS_STRING_STATE())
+        vt->parser.state = NORMAL;
+      vt->parser.in_esc = TRUE;
       continue;
     }
     else if(c == 0x07 &&  // BEL, can stand for ST in OSC or DCS state
-            vt->parser.state == STRING) {
+            IS_STRING_STATE()) {
       // fallthrough
     }
     else if(c < 0x20) { // other C0
@@ -182,96 +157,72 @@ size_t vterm_input_write(VTerm *vt, const char *bytes, size_t len)
           if(pos + 2 < len && bytes[pos + 1] == 0x20 && bytes[pos + 2] == 0x08)
             vt->in_backspace = 2; // Trigger when count down to 1
       }
-      if(vt->parser.state >= STRING)
-        more_string(vt, string_start, bytes + pos - string_start);
+      if(IS_STRING_STATE())
+        string_fragment(vt, string_start, bytes + pos - string_start, FALSE);
       do_control(vt, c);
-      if(vt->parser.state >= STRING)
+      if(IS_STRING_STATE())
         string_start = bytes + pos + 1;
       continue;
     }
     // else fallthrough
 
+    string_len = bytes + pos - string_start;
+
+    if(vt->parser.in_esc) {
+      // Hoist an ESC letter into a C1 if we're not in a string mode
+      // Always accept ESC \ == ST even in string mode
+      if(!vt->parser.intermedlen &&
+          c >= 0x40 && c < 0x60 &&
+          ((!IS_STRING_STATE() || c == 0x5c))) {
+        c += 0x40;
+        c1_allowed = TRUE;
+        string_len -= 1;
+        vt->parser.in_esc = FALSE;
+      }
+      else {
+        string_start = NULL;
+        vt->parser.state = NORMAL;
+      }
+    }
+
     switch(vt->parser.state) {
-    case ESC_IN_STRING:
-      if(c == 0x5c) { // ST
-        vt->parser.state = STRING;
-        done_string(vt, string_start, bytes + pos - string_start - 1);
-        ENTER_NORMAL_STATE();
-        break;
-      }
-      vt->parser.state = ESC;
-      // else fallthrough
-
-    case ESC:
-      switch(c) {
-      case 0x50: // DCS
-        start_string(vt, VTERM_PARSER_DCS);
-        ENTER_STRING_STATE();
-        break;
-      case 0x5b: // CSI
-        vt->parser.csi_leaderlen = 0;
-        ENTER_STATE(CSI_LEADER);
-        break;
-      case 0x5d: // OSC
-        start_string(vt, VTERM_PARSER_OSC);
-        ENTER_STRING_STATE();
-        break;
-      default:
-        if(is_intermed(c)) {
-          if(vt->parser.intermedlen < INTERMED_MAX-1)
-            vt->parser.intermed[vt->parser.intermedlen++] = c;
-        }
-        else if(!vt->parser.intermedlen && c >= 0x40 && c < 0x60) {
-          do_control(vt, c + 0x40);
-          ENTER_NORMAL_STATE();
-        }
-        else if(c >= 0x30 && c < 0x7f) {
-          do_escape(vt, c);
-          ENTER_NORMAL_STATE();
-        }
-        else {
-          DEBUG_LOG1("TODO: Unhandled byte %02x in Escape\n", c);
-        }
-      }
-      break;
-
     case CSI_LEADER:
       // Extract leader bytes 0x3c to 0x3f
       if(c >= 0x3c && c <= 0x3f) {
-        if(vt->parser.csi_leaderlen < CSI_LEADER_MAX-1)
-          vt->parser.csi_leader[vt->parser.csi_leaderlen++] = c;
+        if(vt->parser.v.csi.leaderlen < CSI_LEADER_MAX-1)
+          vt->parser.v.csi.leader[vt->parser.v.csi.leaderlen++] = c;
         break;
       }
 
       // else fallthrough
-      vt->parser.csi_leader[vt->parser.csi_leaderlen] = 0;
+      vt->parser.v.csi.leader[vt->parser.v.csi.leaderlen] = 0;
 
-      vt->parser.csi_argi = 0;
-      vt->parser.csi_args[0] = CSI_ARG_MISSING;
+      vt->parser.v.csi.argi = 0;
+      vt->parser.v.csi.args[0] = CSI_ARG_MISSING;
       vt->parser.state = CSI_ARGS;
 
       // fallthrough
     case CSI_ARGS:
       // Numerical value of argument
       if(c >= '0' && c <= '9') {
-        if(vt->parser.csi_args[vt->parser.csi_argi] == CSI_ARG_MISSING)
-          vt->parser.csi_args[vt->parser.csi_argi] = 0;
-        vt->parser.csi_args[vt->parser.csi_argi] *= 10;
-        vt->parser.csi_args[vt->parser.csi_argi] += c - '0';
+        if(vt->parser.v.csi.args[vt->parser.v.csi.argi] == CSI_ARG_MISSING)
+          vt->parser.v.csi.args[vt->parser.v.csi.argi] = 0;
+        vt->parser.v.csi.args[vt->parser.v.csi.argi] *= 10;
+        vt->parser.v.csi.args[vt->parser.v.csi.argi] += c - '0';
         break;
       }
       if(c == ':') {
-        vt->parser.csi_args[vt->parser.csi_argi] |= CSI_ARG_FLAG_MORE;
+        vt->parser.v.csi.args[vt->parser.v.csi.argi] |= CSI_ARG_FLAG_MORE;
         c = ';';
       }
       if(c == ';') {
-        vt->parser.csi_argi++;
-        vt->parser.csi_args[vt->parser.csi_argi] = CSI_ARG_MISSING;
+        vt->parser.v.csi.argi++;
+        vt->parser.v.csi.args[vt->parser.v.csi.argi] = CSI_ARG_MISSING;
         break;
       }
 
       // else fallthrough
-      vt->parser.csi_argi++;
+      vt->parser.v.csi.argi++;
       vt->parser.intermedlen = 0;
       vt->parser.state = CSI_INTERMED;
       // fallthrough
@@ -293,31 +244,79 @@ size_t vterm_input_write(VTerm *vt, const char *bytes, size_t len)
       ENTER_NORMAL_STATE();
       break;
 
-    case STRING:
-      if(c == 0x07 || (c == 0x9c && !vt->mode.utf8)) {
-        done_string(vt, string_start, bytes + pos - string_start);
-        ENTER_NORMAL_STATE();
+    case OSC_COMMAND:
+      /* Numerical value of command */
+      if(c >= '0' && c <= '9') {
+        if(vt->parser.v.osc.command == -1)
+          vt->parser.v.osc.command = 0;
+        else
+          vt->parser.v.osc.command *= 10;
+        vt->parser.v.osc.command += c - '0';
+        break;
       }
-      else if (pos + 1 == len) {
-	// end of input but OSC string isn't finished yet, copy it to
-	// vt->parser.strbuffer to continue it later
-        more_string(vt, string_start, bytes + pos + 1 - string_start);
+      if(c == ';') {
+        vt->parser.state = OSC;
+        string_start = bytes + pos + 1;
+        break;
+      }
+
+      /* else fallthrough */
+      string_start = bytes + pos;
+      string_len   = 0;
+      vt->parser.state = OSC;
+      goto string_state;
+
+    case DCS_COMMAND:
+      if(vt->parser.v.dcs.commandlen < CSI_LEADER_MAX)
+        vt->parser.v.dcs.command[vt->parser.v.dcs.commandlen++] = c;
+
+      if(c >= 0x40 && c<= 0x7e) {
+        string_start = bytes + pos + 1;
+        vt->parser.state = DCS;
+      }
+      break;
+
+string_state:
+    case OSC:
+    case DCS:
+      if(c == 0x07 || (c1_allowed && c == 0x9c)) {
+        string_fragment(vt, string_start, string_len, TRUE);
+        ENTER_NORMAL_STATE();
       }
       break;
 
     case NORMAL:
-      if(c >= 0x80 && c < 0xa0 && !vt->mode.utf8) {
+      if(vt->parser.in_esc) {
+        if(is_intermed(c)) {
+          if(vt->parser.intermedlen < INTERMED_MAX-1)
+            vt->parser.intermed[vt->parser.intermedlen++] = c;
+        }
+        else if(c >= 0x30 && c < 0x7f) {
+          do_escape(vt, c);
+          vt->parser.in_esc = 0;
+          ENTER_NORMAL_STATE();
+        }
+        else {
+          DEBUG_LOG1("TODO: Unhandled byte %02x in Escape\n", c);
+        }
+        break;
+      }
+      if(c1_allowed && c >= 0x80 && c < 0xa0) {
         switch(c) {
         case 0x90: // DCS
-          start_string(vt, VTERM_PARSER_DCS);
-          ENTER_STRING_STATE();
+          vt->parser.string_initial = TRUE;
+          vt->parser.v.dcs.commandlen = 0;
+          ENTER_STATE(DCS_COMMAND);
           break;
         case 0x9b: // CSI
+          vt->parser.v.csi.leaderlen = 0;
           ENTER_STATE(CSI_LEADER);
           break;
         case 0x9d: // OSC
-          start_string(vt, VTERM_PARSER_OSC);
-          ENTER_STRING_STATE();
+          vt->parser.v.osc.command = -1;
+          vt->parser.string_initial = TRUE;
+          string_start = bytes + pos + 1;
+          ENTER_STATE(OSC_COMMAND);
           break;
         default:
           do_control(vt, c);
@@ -341,6 +340,9 @@ size_t vterm_input_write(VTerm *vt, const char *bytes, size_t len)
     }
   }
 
+  if(string_start)
+    string_fragment(vt, string_start, bytes + pos - string_start, FALSE);
+
   return len;
 }
 

src/libvterm/src/pen.c

@@ -170,6 +170,7 @@ INTERNAL void vterm_state_resetpen(VTermState *state)
   state->pen.italic = 0;    setpenattr_bool(state, VTERM_ATTR_ITALIC, 0);
   state->pen.blink = 0;     setpenattr_bool(state, VTERM_ATTR_BLINK, 0);
   state->pen.reverse = 0;   setpenattr_bool(state, VTERM_ATTR_REVERSE, 0);
+  state->pen.conceal = 0;   setpenattr_bool(state, VTERM_ATTR_CONCEAL, 0);
   state->pen.strike = 0;    setpenattr_bool(state, VTERM_ATTR_STRIKE, 0);
   state->pen.font = 0;      setpenattr_int( state, VTERM_ATTR_FONT, 0);
 
@@ -192,6 +193,7 @@ INTERNAL void vterm_state_savepen(VTermState *state, int save)
     setpenattr_bool(state, VTERM_ATTR_ITALIC,     state->pen.italic);
     setpenattr_bool(state, VTERM_ATTR_BLINK,      state->pen.blink);
     setpenattr_bool(state, VTERM_ATTR_REVERSE,    state->pen.reverse);
+    setpenattr_bool(state, VTERM_ATTR_CONCEAL,    state->pen.conceal);
     setpenattr_bool(state, VTERM_ATTR_STRIKE,     state->pen.strike);
     setpenattr_int( state, VTERM_ATTR_FONT,       state->pen.font);
     setpenattr_col( state, VTERM_ATTR_FOREGROUND, state->pen.fg);
@@ -261,9 +263,26 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
       setpenattr_bool(state, VTERM_ATTR_ITALIC, 1);
       break;
 
-    case 4: // Underline single
-      state->pen.underline = 1;
-      setpenattr_int(state, VTERM_ATTR_UNDERLINE, 1);
+    case 4: // Underline
+      state->pen.underline = VTERM_UNDERLINE_SINGLE;
+      if(CSI_ARG_HAS_MORE(args[argi])) {
+        argi++;
+        switch(CSI_ARG(args[argi])) {
+          case 0:
+            state->pen.underline = 0;
+            break;
+          case 1:
+            state->pen.underline = VTERM_UNDERLINE_SINGLE;
+            break;
+          case 2:
+            state->pen.underline = VTERM_UNDERLINE_DOUBLE;
+            break;
+          case 3:
+            state->pen.underline = VTERM_UNDERLINE_CURLY;
+            break;
+        }
+      }
+      setpenattr_int(state, VTERM_ATTR_UNDERLINE, state->pen.underline);
       break;
 
     case 5: // Blink
@@ -276,6 +295,11 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
       setpenattr_bool(state, VTERM_ATTR_REVERSE, 1);
       break;
 
+    case 8: // Conceal on
+      state->pen.conceal = 1;
+      setpenattr_bool(state, VTERM_ATTR_CONCEAL, 1);
+      break;
+
     case 9: // Strikethrough on
       state->pen.strike = 1;
       setpenattr_bool(state, VTERM_ATTR_STRIKE, 1);
@@ -288,8 +312,8 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
       break;
 
     case 21: // Underline double
-      state->pen.underline = 2;
-      setpenattr_int(state, VTERM_ATTR_UNDERLINE, 2);
+      state->pen.underline = VTERM_UNDERLINE_DOUBLE;
+      setpenattr_int(state, VTERM_ATTR_UNDERLINE, state->pen.underline);
       break;
 
     case 22: // Bold off
@@ -317,6 +341,11 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
       setpenattr_bool(state, VTERM_ATTR_REVERSE, 0);
       break;
 
+    case 28: // Conceal off (Reveal)
+      state->pen.conceal = 0;
+      setpenattr_bool(state, VTERM_ATTR_CONCEAL, 0);
+      break;
+
     case 29: // Strikethrough off
       state->pen.strike = 0;
       setpenattr_bool(state, VTERM_ATTR_STRIKE, 0);
@@ -405,8 +434,10 @@ INTERNAL int vterm_state_getpen(VTermState *state, long args[], int argcount UNU
   if(state->pen.italic)
     args[argi++] = 3;
 
-  if(state->pen.underline == 1)
+  if(state->pen.underline == VTERM_UNDERLINE_SINGLE)
     args[argi++] = 4;
+  if(state->pen.underline == VTERM_UNDERLINE_CURLY)
+    args[argi++] = 4 | CSI_ARG_FLAG_MORE, args[argi++] = 3;
 
   if(state->pen.blink)
     args[argi++] = 5;
@@ -414,13 +445,16 @@ INTERNAL int vterm_state_getpen(VTermState *state, long args[], int argcount UNU
   if(state->pen.reverse)
     args[argi++] = 7;
 
+  if(state->pen.conceal)
+    args[argi++] = 8;
+
   if(state->pen.strike)
     args[argi++] = 9;
 
   if(state->pen.font)
     args[argi++] = 10 + state->pen.font;
 
-  if(state->pen.underline == 2)
+  if(state->pen.underline == VTERM_UNDERLINE_DOUBLE)
     args[argi++] = 21;
 
   if(state->fg_index >= 0 && state->fg_index < 8)
@@ -493,6 +527,10 @@ int vterm_state_get_penattr(const VTermState *state, VTermAttr attr, VTermValue
     val->boolean = state->pen.reverse;
     return 1;
 
+  case VTERM_ATTR_CONCEAL:
+    val->boolean = state->pen.conceal;
+    return 1;
+
   case VTERM_ATTR_STRIKE:
     val->boolean = state->pen.strike;
     return 1;

src/libvterm/src/screen.c

@@ -21,6 +21,7 @@ typedef struct
   unsigned int italic    : 1;
   unsigned int blink     : 1;
   unsigned int reverse   : 1;
+  unsigned int conceal   : 1;
   unsigned int strike    : 1;
   unsigned int font      : 4; // 0 to 9
 
@@ -37,8 +38,6 @@ typedef struct
   ScreenPen pen;
 } ScreenCell;
 
-static int vterm_screen_set_cell(VTermScreen *screen, VTermPos pos, const VTermScreenCell *cell);
-
 struct VTermScreen
 {
   VTerm *vt;
@@ -69,6 +68,12 @@ struct VTermScreen
   ScreenPen pen;
 };
 
+static void clearcell(const VTermScreen *screen, ScreenCell *cell)
+{
+  cell->chars[0] = 0;
+  cell->pen = screen->pen;
+}
+
 static ScreenCell *getcell(const VTermScreen *screen, int row, int col)
 {
   if(row < 0 || row >= screen->rows)
@@ -80,28 +85,18 @@ static ScreenCell *getcell(const VTermScreen *screen, int row, int col)
   return screen->buffer + (screen->cols * row) + col;
 }
 
-static ScreenCell *realloc_buffer(VTermScreen *screen, ScreenCell *buffer, int new_rows, int new_cols)
+static ScreenCell *alloc_buffer(VTermScreen *screen, int rows, int cols)
 {
-  ScreenCell *new_buffer = vterm_allocator_malloc(screen->vt, sizeof(ScreenCell) * new_rows * new_cols);
-  int row, col;
+  ScreenCell *new_buffer = vterm_allocator_malloc(screen->vt, sizeof(ScreenCell) * rows * cols);
+  int row;
+  int col;
 
-  if (new_buffer == NULL)
-    return NULL;
-  for(row = 0; row < new_rows; row++) {
-    for(col = 0; col < new_cols; col++) {
-      ScreenCell *new_cell = new_buffer + row*new_cols + col;
-
-      if(buffer && row < screen->rows && col < screen->cols)
-        *new_cell = buffer[row * screen->cols + col];
-      else {
-        new_cell->chars[0] = 0;
-        new_cell->pen = screen->pen;
-      }
+  for(row = 0; row < rows; row++) {
+    for(col = 0; col < cols; col++) {
+      clearcell(screen, &new_buffer[row * cols + col]);
     }
   }
 
-  vterm_allocator_free(screen->vt, buffer);
-
   return new_buffer;
 }
 
@@ -207,21 +202,27 @@ static int putglyph(VTermGlyphInfo *info, VTermPos pos, void *user)
   return 1;
 }
 
+static void sb_pushline_from_row(VTermScreen *screen, int row)
+{
+  VTermPos pos;
+  pos.row = row;
+  for(pos.col = 0; pos.col < screen->cols; pos.col++)
+    vterm_screen_get_cell(screen, pos, screen->sb_buffer + pos.col);
+
+  (screen->callbacks->sb_pushline)(screen->cols, screen->sb_buffer, screen->cbdata);
+}
+
 static int moverect_internal(VTermRect dest, VTermRect src, void *user)
 {
   VTermScreen *screen = user;
 
   if(screen->callbacks && screen->callbacks->sb_pushline &&
-     dest.start_row == 0 && dest.start_col == 0 &&  // starts top-left corner
-     dest.end_col == screen->cols &&                // full width
-     screen->buffer == screen->buffers[0]) {        // not altscreen
-    VTermPos pos;
-    for(pos.row = 0; pos.row < src.start_row; pos.row++) {
-      for(pos.col = 0; pos.col < screen->cols; pos.col++)
-        (void)vterm_screen_get_cell(screen, pos, screen->sb_buffer + pos.col);
-
-      (screen->callbacks->sb_pushline)(screen->cols, screen->sb_buffer, screen->cbdata);
-    }
+     dest.start_row == 0 && dest.start_col == 0 &&        // starts top-left corner
+     dest.end_col == screen->cols &&                      // full width
+     screen->buffer == screen->buffers[BUFIDX_PRIMARY]) { // not altscreen
+    int row;
+    for(row = 0; row < src.start_row; row++)
+      sb_pushline_from_row(screen, row);
   }
 
   {
@@ -420,6 +421,9 @@ static int setpenattr(VTermAttr attr, VTermValue *val, void *user)
   case VTERM_ATTR_REVERSE:
     screen->pen.reverse = val->boolean;
     return 1;
+  case VTERM_ATTR_CONCEAL:
+    screen->pen.conceal = val->boolean;
+    return 1;
   case VTERM_ATTR_STRIKE:
     screen->pen.strike = val->boolean;
     return 1;
@@ -446,10 +450,10 @@ static int settermprop(VTermProp prop, VTermValue *val, void *user)
 
   switch(prop) {
   case VTERM_PROP_ALTSCREEN:
-    if(val->boolean && !screen->buffers[1])
+    if(val->boolean && !screen->buffers[BUFIDX_ALTSCREEN])
       return 0;
 
-    screen->buffer = val->boolean ? screen->buffers[1] : screen->buffers[0];
+    screen->buffer = val->boolean ? screen->buffers[BUFIDX_ALTSCREEN] : screen->buffers[BUFIDX_PRIMARY];
     // only send a damage event on disable; because during enable there's an
     // erase that sends a damage anyway
     if(!val->boolean)
@@ -479,95 +483,146 @@ static int bell(void *user)
   return 0;
 }
 
-static int resize(int new_rows, int new_cols, VTermPos *delta, void *user)
+static void resize_buffer(VTermScreen *screen, int bufidx, int new_rows, int new_cols, int active, VTermStateFields *statefields)
 {
-  VTermScreen *screen = user;
-
-  int is_altscreen = (screen->buffers[1] && screen->buffer == screen->buffers[1]);
-
   int old_rows = screen->rows;
   int old_cols = screen->cols;
-  int first_blank_row;
 
-  if(!is_altscreen && new_rows < old_rows) {
-    // Fewer rows - determine if we're going to scroll at all, and if so, push
-    // those lines to scrollback
-    VTermPos pos = { 0, 0 };
-    VTermPos cursor = screen->state->pos;
-    // Find the first blank row after the cursor.
-    for(pos.row = old_rows - 1; pos.row >= new_rows; pos.row--)
-      if(!vterm_screen_is_eol(screen, pos) || cursor.row == pos.row)
-        break;
+  ScreenCell *old_buffer = screen->buffers[bufidx];
+  ScreenCell *new_buffer = vterm_allocator_malloc(screen->vt, sizeof(ScreenCell) * new_rows * new_cols);
 
-    first_blank_row = pos.row + 1;
-    if(first_blank_row > new_rows) {
-      VTermRect rect = {0,0,0,0};
-      rect.end_row   = old_rows;
-      rect.end_col   = old_cols;
-      scrollrect(rect, first_blank_row - new_rows, 0, user);
-      vterm_screen_flush_damage(screen);
+  /* Find the final row of old buffer content */
+  int old_row = old_rows - 1;
+  int new_row = new_rows - 1;
+  int col;
 
-      delta->row -= first_blank_row - new_rows;
+  while(new_row >= 0 && old_row >= 0) {
+    for(col = 0; col < old_cols && col < new_cols; col++)
+      new_buffer[new_row * new_cols + col] = old_buffer[old_row * old_cols + col];
+    for( ; col < new_cols; col++)
+      clearcell(screen, &new_buffer[new_row * new_cols + col]);
+
+    old_row--;
+    new_row--;
+
+    if(new_row < 0 && old_row >= 0 &&
+        new_buffer[(new_rows - 1) * new_cols].chars[0] == 0 &&
+        (!active || statefields->pos.row < (new_rows - 1))) {
+      int moverows = new_rows - 1;
+      memmove(&new_buffer[1 * new_cols], &new_buffer[0], moverows * new_cols * sizeof(ScreenCell));
+
+      new_row++;
     }
   }
 
-  screen->buffers[0] = realloc_buffer(screen, screen->buffers[0], new_rows, new_cols);
-  if(screen->buffers[1])
-    screen->buffers[1] = realloc_buffer(screen, screen->buffers[1], new_rows, new_cols);
+  if(old_row >= 0 && bufidx == BUFIDX_PRIMARY) {
+    /* Push spare lines to scrollback buffer */
+    int row;
+    for(row = 0; row <= old_row; row++)
+      sb_pushline_from_row(screen, row);
+    if(active)
+      statefields->pos.row -= (old_row + 1);
+  }
+  if(new_row >= 0 && bufidx == BUFIDX_PRIMARY &&
+      screen->callbacks && screen->callbacks->sb_popline) {
+    /* Try to backfill rows by popping scrollback buffer */
+    while(new_row >= 0) {
+      VTermPos pos;
+      if(!(screen->callbacks->sb_popline(old_cols, screen->sb_buffer, screen->cbdata)))
+        break;
 
-  screen->buffer = is_altscreen ? screen->buffers[1] : screen->buffers[0];
+      pos.row = new_row;
+      for(pos.col = 0; pos.col < old_cols && pos.col < new_cols; pos.col += screen->sb_buffer[pos.col].width) {
+        VTermScreenCell *src = &screen->sb_buffer[pos.col];
+        ScreenCell *dst = &new_buffer[pos.row * new_cols + pos.col];
+	int i;
+
+        for(i = 0; i < VTERM_MAX_CHARS_PER_CELL; i++) {
+          dst->chars[i] = src->chars[i];
+          if(!src->chars[i])
+            break;
+        }
+
+        dst->pen.bold      = src->attrs.bold;
+        dst->pen.underline = src->attrs.underline;
+        dst->pen.italic    = src->attrs.italic;
+        dst->pen.blink     = src->attrs.blink;
+        dst->pen.reverse   = src->attrs.reverse ^ screen->global_reverse;
+        dst->pen.conceal   = src->attrs.conceal;
+        dst->pen.strike    = src->attrs.strike;
+        dst->pen.font      = src->attrs.font;
+
+        dst->pen.fg = src->fg;
+        dst->pen.bg = src->bg;
+
+        if(src->width == 2 && pos.col < (new_cols-1))
+          (dst + 1)->chars[0] = (uint32_t) -1;
+      }
+      for( ; pos.col < new_cols; pos.col++)
+        clearcell(screen, &new_buffer[pos.row * new_cols + pos.col]);
+      new_row--;
+
+      if(active)
+        statefields->pos.row++;
+    }
+  }
+
+  if(new_row >= 0) {
+    /* Scroll new rows back up to the top and fill in blanks at the bottom */
+    int moverows = new_rows - new_row - 1;
+    memmove(&new_buffer[0], &new_buffer[(new_row + 1) * new_cols], moverows * new_cols * sizeof(ScreenCell));
+
+    for(new_row = moverows; new_row < new_rows; new_row++)
+    {
+      for(col = 0; col < new_cols; col++)
+        clearcell(screen, &new_buffer[new_row * new_cols + col]);
+    }
+  }
+
+  vterm_allocator_free(screen->vt, old_buffer);
+  screen->buffers[bufidx] = new_buffer;
+
+  return;
+
+  /* REFLOW TODO:
+   *   Handle delta. Probably needs to be a full cursorpos that we edit
+   */
+}
+
+static int resize(int new_rows, int new_cols, VTermStateFields *fields, void *user)
+{
+  VTermScreen *screen = user;
+
+  int altscreen_active = (screen->buffers[BUFIDX_ALTSCREEN] && screen->buffer == screen->buffers[BUFIDX_ALTSCREEN]);
+
+  int old_cols = screen->cols;
+
+  if(new_cols > old_cols) {
+    /* Ensure that ->sb_buffer is large enough for a new or and old row */
+    if(screen->sb_buffer)
+      vterm_allocator_free(screen->vt, screen->sb_buffer);
+
+    screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols);
+  }
+
+  resize_buffer(screen, 0, new_rows, new_cols, !altscreen_active, fields);
+  if(screen->buffers[BUFIDX_ALTSCREEN])
+    resize_buffer(screen, 1, new_rows, new_cols, altscreen_active, fields);
+
+  screen->buffer = altscreen_active ? screen->buffers[BUFIDX_ALTSCREEN] : screen->buffers[BUFIDX_PRIMARY];
 
   screen->rows = new_rows;
   screen->cols = new_cols;
 
-  vterm_allocator_free(screen->vt, screen->sb_buffer);
+  if(new_cols <= old_cols) {
+    if(screen->sb_buffer)
+      vterm_allocator_free(screen->vt, screen->sb_buffer);
 
-  screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols);
-
-  if(new_cols > old_cols) {
-    VTermRect rect;
-    rect.start_row = 0;
-    rect.end_row   = old_rows;
-    rect.start_col = old_cols;
-    rect.end_col   = new_cols;
-    damagerect(screen, rect);
+    screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols);
   }
 
-  if(new_rows > old_rows) {
-    if(!is_altscreen && screen->callbacks && screen->callbacks->sb_popline) {
-      int rows = new_rows - old_rows;
-      while(rows) {
-        VTermRect rect = {0,0,0,0};
-        VTermPos pos = { 0, 0 };
-        if(!(screen->callbacks->sb_popline(screen->cols, screen->sb_buffer, screen->cbdata)))
-          break;
-
-	rect.end_row   = screen->rows;
-	rect.end_col   = screen->cols;
-        scrollrect(rect, -1, 0, user);
-
-        for(pos.col = 0; pos.col < screen->cols; pos.col += screen->sb_buffer[pos.col].width)
-          vterm_screen_set_cell(screen, pos, screen->sb_buffer + pos.col);
-
-        rect.end_row = 1;
-        damagerect(screen, rect);
-
-        vterm_screen_flush_damage(screen);
-
-        rows--;
-        delta->row++;
-      }
-    }
-
-    {
-      VTermRect rect;
-      rect.start_row = old_rows;
-      rect.end_row   = new_rows;
-      rect.start_col = 0;
-      rect.end_col   = new_cols;
-      damagerect(screen, rect);
-    }
-  }
+  /* TODO: Maaaaybe we can optimise this if there's no reflow happening */
+  damagescreen(screen);
 
   if(screen->callbacks && screen->callbacks->resize)
     return (*screen->callbacks->resize)(new_rows, new_cols, screen->cbdata);
@@ -651,8 +706,10 @@ static VTermScreen *screen_new(VTerm *vt)
   screen->callbacks = NULL;
   screen->cbdata    = NULL;
 
-  screen->buffers[0] = realloc_buffer(screen, NULL, rows, cols);
-  screen->buffer = screen->buffers[0];
+  screen->buffers[BUFIDX_PRIMARY] = alloc_buffer(screen, rows, cols);
+
+  screen->buffer = screen->buffers[BUFIDX_PRIMARY];
+
   screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * cols);
   if (screen->buffer == NULL || screen->sb_buffer == NULL)
   {
@@ -667,8 +724,10 @@ static VTermScreen *screen_new(VTerm *vt)
 
 INTERNAL void vterm_screen_free(VTermScreen *screen)
 {
-  vterm_allocator_free(screen->vt, screen->buffers[0]);
-  vterm_allocator_free(screen->vt, screen->buffers[1]);
+  vterm_allocator_free(screen->vt, screen->buffers[BUFIDX_PRIMARY]);
+  if(screen->buffers[BUFIDX_ALTSCREEN])
+    vterm_allocator_free(screen->vt, screen->buffers[BUFIDX_ALTSCREEN]);
+
   vterm_allocator_free(screen->vt, screen->sb_buffer);
   vterm_allocator_free(screen->vt, screen);
 }
@@ -752,7 +811,7 @@ int vterm_screen_get_cell(const VTermScreen *screen, VTermPos pos, VTermScreenCe
   if(!intcell)
     return 0;
 
-  for(i = 0; ; i++) {
+  for(i = 0; i < VTERM_MAX_CHARS_PER_CELL; i++) {
     cell->chars[i] = intcell->chars[i];
     if(!intcell->chars[i])
       break;
@@ -763,6 +822,7 @@ int vterm_screen_get_cell(const VTermScreen *screen, VTermPos pos, VTermScreenCe
   cell->attrs.italic    = intcell->pen.italic;
   cell->attrs.blink     = intcell->pen.blink;
   cell->attrs.reverse   = intcell->pen.reverse ^ screen->global_reverse;
+  cell->attrs.conceal   = intcell->pen.conceal;
   cell->attrs.strike    = intcell->pen.strike;
   cell->attrs.font      = intcell->pen.font;
 
@@ -798,41 +858,6 @@ int vterm_screen_get_cell(const VTermScreen *screen, VTermPos pos, VTermScreenCe
   return 1;
 }
 
-/*
- * Copy external to internal representation of a screen cell
- * static because it's only used internally for sb_popline during resize
- */
-static int vterm_screen_set_cell(VTermScreen *screen, VTermPos pos, const VTermScreenCell *cell)
-{
-  ScreenCell *intcell = getcell(screen, pos.row, pos.col);
-  int i;
-
-  if(!intcell)
-    return 0;
-
-  for(i = 0; ; i++) {
-    intcell->chars[i] = cell->chars[i];
-    if(!cell->chars[i])
-      break;
-  }
-
-  intcell->pen.bold      = cell->attrs.bold;
-  intcell->pen.underline = cell->attrs.underline;
-  intcell->pen.italic    = cell->attrs.italic;
-  intcell->pen.blink     = cell->attrs.blink;
-  intcell->pen.reverse   = cell->attrs.reverse ^ screen->global_reverse;
-  intcell->pen.strike    = cell->attrs.strike;
-  intcell->pen.font      = cell->attrs.font;
-
-  intcell->pen.fg = cell->fg;
-  intcell->pen.bg = cell->bg;
-
-  if(cell->width == 2)
-    getcell(screen, pos.row, pos.col + 1)->chars[0] = (uint32_t)-1;
-
-  return 1;
-}
-
 int vterm_screen_is_eol(const VTermScreen *screen, VTermPos pos)
 {
   // This cell is EOL if this and every cell to the right is black
@@ -854,12 +879,11 @@ VTermScreen *vterm_obtain_screen(VTerm *vt)
 
 void vterm_screen_enable_altscreen(VTermScreen *screen, int altscreen)
 {
-
-  if(!screen->buffers[1] && altscreen) {
+  if(!screen->buffers[BUFIDX_ALTSCREEN] && altscreen) {
     int rows, cols;
     vterm_get_size(screen->vt, &rows, &cols);
 
-    screen->buffers[1] = realloc_buffer(screen, NULL, rows, cols);
+    screen->buffers[BUFIDX_ALTSCREEN] = alloc_buffer(screen, rows, cols);
   }
 }
 
@@ -874,7 +898,7 @@ void *vterm_screen_get_cbdata(VTermScreen *screen)
   return screen->cbdata;
 }
 
-void vterm_screen_set_unrecognised_fallbacks(VTermScreen *screen, const VTermParserCallbacks *fallbacks, void *user)
+void vterm_screen_set_unrecognised_fallbacks(VTermScreen *screen, const VTermStateFallbacks *fallbacks, void *user)
 {
   vterm_state_set_unrecognised_fallbacks(screen->state, fallbacks, user);
 }
@@ -919,6 +943,8 @@ static int attrs_differ(VTermAttrMask attrs, ScreenCell *a, ScreenCell *b)
     return 1;
   if((attrs & VTERM_ATTR_REVERSE_MASK)    && (a->pen.reverse != b->pen.reverse))
     return 1;
+  if((attrs & VTERM_ATTR_CONCEAL_MASK)    && (a->pen.conceal != b->pen.conceal))
+    return 1;
   if((attrs & VTERM_ATTR_STRIKE_MASK)     && (a->pen.strike != b->pen.strike))
     return 1;
   if((attrs & VTERM_ATTR_FONT_MASK)       && (a->pen.font != b->pen.font))

src/libvterm/src/state.c

@@ -44,6 +44,15 @@ static void updatecursor(VTermState *state, VTermPos *oldpos, int cancel_phantom
 
 static void erase(VTermState *state, VTermRect rect, int selective)
 {
+  if(rect.end_col == state->cols) {
+    int row;
+    /* If we're erasing the final cells of any lines, cancel the continuation
+     * marker on the subsequent line
+     */
+    for(row = rect.start_row + 1; row < rect.end_row + 1 && row < state->rows; row++)
+      state->lineinfo[row].continuation = 0;
+  }
+
   if(state->callbacks && state->callbacks->erase)
     if((*state->callbacks->erase)(rect, selective, state->cbdata))
       return;
@@ -73,13 +82,29 @@ static VTermState *vterm_state_new(VTerm *vt)
 
   state->bold_is_highbright = 0;
 
+  state->combine_chars_size = 16;
+  state->combine_chars = vterm_allocator_malloc(state->vt, state->combine_chars_size * sizeof(state->combine_chars[0]));
+
+  state->tabstops = vterm_allocator_malloc(state->vt, (state->cols + 7) / 8);
+
+  state->lineinfos[BUFIDX_PRIMARY]   = vterm_allocator_malloc(state->vt, state->rows * sizeof(VTermLineInfo));
+  /* TODO: Make an 'enable' function */
+  state->lineinfos[BUFIDX_ALTSCREEN] = vterm_allocator_malloc(state->vt, state->rows * sizeof(VTermLineInfo));
+  state->lineinfo = state->lineinfos[BUFIDX_PRIMARY];
+
+  state->encoding_utf8.enc = vterm_lookup_encoding(ENC_UTF8, 'u');
+  if(state->encoding_utf8.enc->init)
+    (*state->encoding_utf8.enc->init)(state->encoding_utf8.enc, state->encoding_utf8.data);
+
   return state;
 }
 
 INTERNAL void vterm_state_free(VTermState *state)
 {
   vterm_allocator_free(state->vt, state->tabstops);
-  vterm_allocator_free(state->vt, state->lineinfo);
+  vterm_allocator_free(state->vt, state->lineinfos[BUFIDX_PRIMARY]);
+  if(state->lineinfos[BUFIDX_ALTSCREEN])
+    vterm_allocator_free(state->vt, state->lineinfos[BUFIDX_ALTSCREEN]);
   vterm_allocator_free(state->vt, state->combine_chars);
   vterm_allocator_free(state->vt, state);
 }
@@ -106,15 +131,23 @@ static void scroll(VTermState *state, VTermRect rect, int downward, int rightwar
   // Update lineinfo if full line
   if(rect.start_col == 0 && rect.end_col == state->cols && rightward == 0) {
     int height = rect.end_row - rect.start_row - abs(downward);
+    int row;
+    VTermLineInfo zeroLineInfo = { 0 };
 
-    if(downward > 0)
+    if(downward > 0) {
       memmove(state->lineinfo + rect.start_row,
               state->lineinfo + rect.start_row + downward,
               height * sizeof(state->lineinfo[0]));
-    else
+      for(row = rect.end_row - downward; row < rect.end_row; row++)
+        state->lineinfo[row] = zeroLineInfo;
+    }
+    else {
       memmove(state->lineinfo + rect.start_row - downward,
               state->lineinfo + rect.start_row,
               height * sizeof(state->lineinfo[0]));
+      for(row = rect.start_row; row < rect.start_row - downward; row++)
+        state->lineinfo[row] = zeroLineInfo;
+    }
   }
 
   if(state->callbacks && state->callbacks->scrollrect)
@@ -367,6 +400,7 @@ static int on_text(const char bytes[], size_t len, void *user)
       linefeed(state);
       state->pos.col = 0;
       state->at_phantom = 0;
+      state->lineinfo[state->pos.row].continuation = 1;
     }
 
     if(state->mode.insert) {
@@ -548,19 +582,12 @@ static int settermprop_int(VTermState *state, VTermProp prop, int v)
   return vterm_state_set_termprop(state, prop, &val);
 }
 
-static int settermprop_string(VTermState *state, VTermProp prop, const char *str, size_t len)
+static int settermprop_string(VTermState *state, VTermProp prop, VTermStringFragment frag)
 {
-  char *strvalue;
-  int r;
   VTermValue val;
-  strvalue = vterm_allocator_malloc(state->vt, (len+1) * sizeof(char));
-  strncpy(strvalue, str, len);
-  strvalue[len] = 0;
 
-  val.string = strvalue;
-  r = vterm_state_set_termprop(state, prop, &val);
-  vterm_allocator_free(state->vt, strvalue);
-  return r;
+  val.string = frag;
+  return vterm_state_set_termprop(state, prop, &val);
 }
 
 static void savecursor(VTermState *state, int save)
@@ -1451,6 +1478,14 @@ static int on_csi(const char *leader, const long args[], int argcount, const cha
       state->scrollregion_bottom = -1;
     }
 
+    // Setting the scrolling region restores the cursor to the home position
+    state->pos.row = 0;
+    state->pos.col = 0;
+    if(state->mode.origin) {
+      state->pos.row += state->scrollregion_top;
+      state->pos.col += SCROLLREGION_LEFT(state);
+    }
+
     break;
 
   case 0x73: // DECSLRM - DEC custom
@@ -1472,6 +1507,14 @@ static int on_csi(const char *leader, const long args[], int argcount, const cha
       state->scrollregion_right = -1;
     }
 
+    // Setting the scrolling region restores the cursor to the home position
+    state->pos.row = 0;
+    state->pos.col = 0;
+    if(state->mode.origin) {
+      state->pos.row += state->scrollregion_top;
+      state->pos.col += SCROLLREGION_LEFT(state);
+    }
+
     break;
 
   case 0x74:
@@ -1568,100 +1611,121 @@ static int on_csi(const char *leader, const long args[], int argcount, const cha
   return 1;
 }
 
-static int on_osc(const char *command, size_t cmdlen, void *user)
+static int on_osc(int command, VTermStringFragment frag, void *user)
 {
   VTermState *state = user;
 
-  if(cmdlen < 2)
-    return 0;
-
-  if(strneq(command, "0;", 2)) {
-    settermprop_string(state, VTERM_PROP_ICONNAME, command + 2, cmdlen - 2);
-    settermprop_string(state, VTERM_PROP_TITLE, command + 2, cmdlen - 2);
-    return 1;
-  }
-  else if(strneq(command, "1;", 2)) {
-    settermprop_string(state, VTERM_PROP_ICONNAME, command + 2, cmdlen - 2);
-    return 1;
-  }
-  else if(strneq(command, "2;", 2)) {
-    settermprop_string(state, VTERM_PROP_TITLE, command + 2, cmdlen - 2);
-    return 1;
-  }
-  else if(strneq(command, "10;", 3)) {
-    // request foreground color: <Esc>]10;?<0x07>
-    int red = state->default_fg.red;
-    int blue = state->default_fg.blue;
-    int green = state->default_fg.green;
-    vterm_push_output_sprintf_ctrl(state->vt, C1_OSC, "10;rgb:%02x%02x/%02x%02x/%02x%02x\x07", red, red, green, green, blue, blue);
-    return 1;
-  }
-  else if(strneq(command, "11;", 3)) {
-    // request background color: <Esc>]11;?<0x07>
-    int red = state->default_bg.red;
-    int blue = state->default_bg.blue;
-    int green = state->default_bg.green;
-    vterm_push_output_sprintf_ctrl(state->vt, C1_OSC, "11;rgb:%02x%02x/%02x%02x/%02x%02x\x07", red, red, green, green, blue, blue);
-    return 1;
-  }
-  else if(strneq(command, "12;", 3)) {
-    settermprop_string(state, VTERM_PROP_CURSORCOLOR, command + 3, cmdlen - 3);
-    return 1;
-  }
-  else if(state->fallbacks && state->fallbacks->osc)
-    if((*state->fallbacks->osc)(command, cmdlen, state->fbdata))
+  switch(command) {
+    case 0:
+      settermprop_string(state, VTERM_PROP_ICONNAME, frag);
+      settermprop_string(state, VTERM_PROP_TITLE, frag);
       return 1;
 
+    case 1:
+      settermprop_string(state, VTERM_PROP_ICONNAME, frag);
+      return 1;
+
+    case 2:
+      settermprop_string(state, VTERM_PROP_TITLE, frag);
+      return 1;
+
+    case 10:
+      {
+        // request foreground color: <Esc>]10;?<0x07>
+        int red = state->default_fg.red;
+        int blue = state->default_fg.blue;
+        int green = state->default_fg.green;
+        vterm_push_output_sprintf_ctrl(state->vt, C1_OSC, "10;rgb:%02x%02x/%02x%02x/%02x%02x\x07", red, red, green, green, blue, blue);
+        return 1;
+      }
+
+    case 11:
+      {
+	// request background color: <Esc>]11;?<0x07>
+	int red = state->default_bg.red;
+	int blue = state->default_bg.blue;
+	int green = state->default_bg.green;
+	vterm_push_output_sprintf_ctrl(state->vt, C1_OSC, "11;rgb:%02x%02x/%02x%02x/%02x%02x\x07", red, red, green, green, blue, blue);
+	return 1;
+      }
+    case 12:
+      settermprop_string(state, VTERM_PROP_CURSORCOLOR, frag);
+      return 1;
+
+    default:
+      if(state->fallbacks && state->fallbacks->osc)
+        if((*state->fallbacks->osc)(command, frag, state->fbdata))
+          return 1;
+  }
+
   return 0;
 }
 
-static void request_status_string(VTermState *state, const char *command, size_t cmdlen)
+static void request_status_string(VTermState *state, VTermStringFragment frag)
 {
   VTerm *vt = state->vt;
 
-  if(cmdlen == 1)
-    switch(command[0]) {
-      case 'm': // Query SGR
-        {
-          long args[20];
-          int argc = vterm_state_getpen(state, args, sizeof(args)/sizeof(args[0]));
-	  int argi;
-          size_t cur = 0;
+  char *tmp = state->tmp.decrqss;
+  size_t i = 0;
 
-          cur += SNPRINTF(vt->tmpbuffer + cur, vt->tmpbuffer_len - cur,
-              vt->mode.ctrl8bit ? "\x90" "1$r" : ESC_S "P" "1$r"); // DCS 1$r ...
-          if(cur >= vt->tmpbuffer_len)
-            return;
+  if(frag.initial)
+    tmp[0] = tmp[1] = tmp[2] = tmp[3] = 0;
 
-          for(argi = 0; argi < argc; argi++) {
-            cur += SNPRINTF(vt->tmpbuffer + cur, vt->tmpbuffer_len - cur,
-                argi == argc - 1             ? "%ld" :
-                CSI_ARG_HAS_MORE(args[argi]) ? "%ld:" :
-                                               "%ld;",
-                 CSI_ARG(args[argi]));
+  while(i < sizeof(state->tmp.decrqss)-1 && tmp[i])
+    i++;
+  while(i < sizeof(state->tmp.decrqss)-1 && frag.len--)
+    tmp[i++] = (frag.str++)[0];
+  tmp[i] = 0;
 
-            if(cur >= vt->tmpbuffer_len)
-              return;
-          }
+  if(!frag.final)
+    return;
 
-          cur += SNPRINTF(vt->tmpbuffer + cur, vt->tmpbuffer_len - cur,
-              vt->mode.ctrl8bit ? "m" "\x9C" : "m" ESC_S "\\"); // ... m ST
-          if(cur >= vt->tmpbuffer_len)
-            return;
+  fprintf(stderr, "DECRQSS on <%s>\n", tmp);
 
-          vterm_push_output_bytes(vt, vt->tmpbuffer, cur);
-        }
+  switch(tmp[0] | tmp[1]<<8 | tmp[2]<<16) {
+    case 'm': {
+      // Query SGR
+      long args[20];
+      int argc = vterm_state_getpen(state, args, sizeof(args)/sizeof(args[0]));
+      size_t cur = 0;
+      int argi;
+
+      cur += SNPRINTF(vt->tmpbuffer + cur, vt->tmpbuffer_len - cur,
+          vt->mode.ctrl8bit ? "\x90" "1$r" : ESC_S "P" "1$r"); // DCS 1$r ...
+      if(cur >= vt->tmpbuffer_len)
         return;
-      case 'r': // Query DECSTBM
-        vterm_push_output_sprintf_dcs(vt, "1$r%d;%dr", state->scrollregion_top+1, SCROLLREGION_BOTTOM(state));
-        return;
-      case 's': // Query DECSLRM
-        vterm_push_output_sprintf_dcs(vt, "1$r%d;%ds", SCROLLREGION_LEFT(state)+1, SCROLLREGION_RIGHT(state));
+
+      for(argi = 0; argi < argc; argi++) {
+        cur += SNPRINTF(vt->tmpbuffer + cur, vt->tmpbuffer_len - cur,
+            argi == argc - 1             ? "%ld" :
+            CSI_ARG_HAS_MORE(args[argi]) ? "%ld:" :
+                                           "%ld;",
+            CSI_ARG(args[argi]));
+        if(cur >= vt->tmpbuffer_len)
+          return;
+      }
+
+      cur += SNPRINTF(vt->tmpbuffer + cur, vt->tmpbuffer_len - cur,
+          vt->mode.ctrl8bit ? "m" "\x9C" : "m" ESC_S "\\"); // ... m ST
+      if(cur >= vt->tmpbuffer_len)
         return;
+
+      vterm_push_output_bytes(vt, vt->tmpbuffer, cur);
+      return;
     }
 
-  if(cmdlen == 2) {
-    if(strneq(command, " q", 2)) {
+    case 'r':
+      // Query DECSTBM
+      vterm_push_output_sprintf_dcs(vt, "1$r%d;%dr", state->scrollregion_top+1, SCROLLREGION_BOTTOM(state));
+      return;
+
+    case 's':
+      // Query DECSLRM
+      vterm_push_output_sprintf_dcs(vt, "1$r%d;%ds", SCROLLREGION_LEFT(state)+1, SCROLLREGION_RIGHT(state));
+      return;
+
+    case ' '|('q'<<8): {
+      // Query DECSCUSR
       int reply;
       switch(state->mode.cursor_shape) {
         case VTERM_PROP_CURSORSHAPE_BLOCK:     reply = 2; break;
@@ -1673,27 +1737,29 @@ static void request_status_string(VTermState *state, const char *command, size_t
       vterm_push_output_sprintf_dcs(vt, "1$r%d q", reply);
       return;
     }
-    else if(strneq(command, "\"q", 2)) {
+
+    case '\"'|('q'<<8):
+      // Query DECSCA
       vterm_push_output_sprintf_dcs(vt, "1$r%d\"q", state->protected_cell ? 1 : 2);
       return;
-    }
   }
 
-  vterm_push_output_sprintf_dcs(state->vt, "0$r%.s", (int)cmdlen, command);
+  vterm_push_output_sprintf_dcs(state->vt, "0$r%s", tmp);
 }
 
-static int on_dcs(const char *command, size_t cmdlen, void *user)
+static int on_dcs(const char *command, size_t commandlen, VTermStringFragment frag, void *user)
 {
   VTermState *state = user;
 
-  if(cmdlen >= 2 && strneq(command, "$q", 2)) {
-    request_status_string(state, command+2, cmdlen-2);
+  if(commandlen == 2 && strneq(command, "$q", 2)) {
+    request_status_string(state, frag);
     return 1;
   }
   else if(state->fallbacks && state->fallbacks->dcs)
-    if((*state->fallbacks->dcs)(command, cmdlen, state->fbdata))
+    if((*state->fallbacks->dcs)(command, commandlen, frag, state->fbdata))
       return 1;
 
+  DEBUG_LOG2("libvterm: Unhandled DCS %.*s\n", (int)commandlen, command);
   return 0;
 }
 
@@ -1701,7 +1767,7 @@ static int on_resize(int rows, int cols, void *user)
 {
   VTermState *state = user;
   VTermPos oldpos = state->pos;
-  VTermPos delta = { 0, 0 };
+  VTermStateFields fields;
 
   if(cols != state->cols) {
     int col;
@@ -1731,22 +1797,30 @@ static int on_resize(int rows, int cols, void *user)
   }
 
   if(rows != state->rows) {
-    int row;
-    VTermLineInfo *newlineinfo = vterm_allocator_malloc(state->vt, rows * sizeof(VTermLineInfo));
-    if (newlineinfo == NULL)
-      return 0;
+    int bufidx;
+    for(bufidx = BUFIDX_PRIMARY; bufidx <= BUFIDX_ALTSCREEN; bufidx++) {
+      int row;
+      VTermLineInfo *oldlineinfo = state->lineinfos[bufidx];
+      VTermLineInfo *newlineinfo;
+      if(!oldlineinfo)
+        continue;
 
-    for(row = 0; row < state->rows && row < rows; row++) {
-      newlineinfo[row] = state->lineinfo[row];
+      newlineinfo = vterm_allocator_malloc(state->vt, rows * sizeof(VTermLineInfo));
+
+      for(row = 0; row < state->rows && row < rows; row++) {
+        newlineinfo[row] = oldlineinfo[row];
+      }
+
+      for( ; row < rows; row++) {
+	VTermLineInfo lineInfo = {0};
+	newlineinfo[row] = lineInfo;
+      }
+
+      vterm_allocator_free(state->vt, state->lineinfos[bufidx]);
+      state->lineinfos[bufidx] = newlineinfo;
     }
 
-    for( ; row < rows; row++) {
-      newlineinfo[row].doublewidth = 0;
-      newlineinfo[row].doubleheight = 0;
-    }
-
-    vterm_allocator_free(state->vt, state->lineinfo);
-    state->lineinfo = newlineinfo;
+    state->lineinfo = state->lineinfos[state->mode.alt_screen ? BUFIDX_ALTSCREEN : BUFIDX_PRIMARY];
   }
 
   state->rows = rows;
@@ -1757,17 +1831,18 @@ static int on_resize(int rows, int cols, void *user)
   if(state->scrollregion_right > -1)
     UBOUND(state->scrollregion_right, state->cols);
 
+  fields.pos = state->pos;
+
   if(state->callbacks && state->callbacks->resize)
-    (*state->callbacks->resize)(rows, cols, &delta, state->cbdata);
+    (*state->callbacks->resize)(rows, cols, &fields, state->cbdata);
+
+  state->pos = fields.pos;
 
   if(state->at_phantom && state->pos.col < cols-1) {
     state->at_phantom = 0;
     state->pos.col++;
   }
 
-  state->pos.row += delta.row;
-  state->pos.col += delta.col;
-
   if(state->pos.row >= rows)
     state->pos.row = rows - 1;
   if(state->pos.col >= cols)
@@ -1803,17 +1878,6 @@ VTermState *vterm_obtain_state(VTerm *vt)
     return NULL;
   vt->state = state;
 
-  state->combine_chars_size = 16;
-  state->combine_chars = vterm_allocator_malloc(state->vt, state->combine_chars_size * sizeof(state->combine_chars[0]));
-
-  state->tabstops = vterm_allocator_malloc(state->vt, (state->cols + 7) / 8);
-
-  state->lineinfo = vterm_allocator_malloc(state->vt, state->rows * sizeof(VTermLineInfo));
-
-  state->encoding_utf8.enc = vterm_lookup_encoding(ENC_UTF8, 'u');
-  if(*state->encoding_utf8.enc->init != NULL)
-    (*state->encoding_utf8.enc->init)(state->encoding_utf8.enc, state->encoding_utf8.data);
-
   vterm_parser_set_callbacks(vt, &parser_callbacks, state);
 
   return state;
@@ -1931,7 +1995,7 @@ void *vterm_state_get_cbdata(VTermState *state)
   return state->cbdata;
 }
 
-void vterm_state_set_unrecognised_fallbacks(VTermState *state, const VTermParserCallbacks *fallbacks, void *user)
+void vterm_state_set_unrecognised_fallbacks(VTermState *state, const VTermStateFallbacks *fallbacks, void *user)
 {
   if(fallbacks) {
     state->fallbacks = fallbacks;
@@ -1976,6 +2040,7 @@ int vterm_state_set_termprop(VTermState *state, VTermProp prop, VTermValue *val)
     return 1;
   case VTERM_PROP_ALTSCREEN:
     state->mode.alt_screen = val->boolean;
+    state->lineinfo = state->lineinfos[state->mode.alt_screen ? BUFIDX_ALTSCREEN : BUFIDX_PRIMARY];
     if(state->mode.alt_screen) {
       VTermRect rect = {0, 0, 0, 0};
       rect.end_row = state->rows;

src/libvterm/src/vterm.c

@@ -55,31 +55,26 @@ VTerm *vterm_new_with_allocator(int rows, int cols, VTermAllocatorFunctions *fun
   vt->parser.callbacks = NULL;
   vt->parser.cbdata    = NULL;
 
-  vt->parser.strbuffer_len = 500; // should be able to hold an OSC string
-  vt->parser.strbuffer_cur = 0;
-  vt->parser.strbuffer = vterm_allocator_malloc(vt, vt->parser.strbuffer_len);
-  if (vt->parser.strbuffer == NULL)
-  {
-    vterm_allocator_free(vt, vt);
-    return NULL;
-  }
-
   vt->outfunc = NULL;
   vt->outdata = NULL;
 
   vt->outbuffer_len = 200;
   vt->outbuffer_cur = 0;
   vt->outbuffer = vterm_allocator_malloc(vt, vt->outbuffer_len);
-  if (vt->outbuffer == NULL)
-  {
-    vterm_allocator_free(vt, vt->parser.strbuffer);
-    vterm_allocator_free(vt, vt);
-    return NULL;
-  }
 
   vt->tmpbuffer_len = 64;
   vt->tmpbuffer = vterm_allocator_malloc(vt, vt->tmpbuffer_len);
 
+  if (vt->tmpbuffer == NULL
+      || vt->outbuffer == NULL
+      || vt->tmpbuffer == NULL)
+  {
+    vterm_allocator_free(vt, vt->outbuffer);
+    vterm_allocator_free(vt, vt->tmpbuffer);
+    vterm_allocator_free(vt, vt);
+    return NULL;
+  }
+
   return vt;
 }
 
@@ -91,8 +86,8 @@ void vterm_free(VTerm *vt)
   if(vt->state)
     vterm_state_free(vt->state);
 
-  vterm_allocator_free(vt, vt->parser.strbuffer);
   vterm_allocator_free(vt, vt->outbuffer);
+  vterm_allocator_free(vt, vt->tmpbuffer);
 
   vterm_allocator_free(vt, vt);
 }
@@ -266,6 +261,7 @@ VTermValueType vterm_get_attr_type(VTermAttr attr)
     case VTERM_ATTR_ITALIC:     return VTERM_VALUETYPE_BOOL;
     case VTERM_ATTR_BLINK:      return VTERM_VALUETYPE_BOOL;
     case VTERM_ATTR_REVERSE:    return VTERM_VALUETYPE_BOOL;
+    case VTERM_ATTR_CONCEAL:    return VTERM_VALUETYPE_BOOL;
     case VTERM_ATTR_STRIKE:     return VTERM_VALUETYPE_BOOL;
     case VTERM_ATTR_FONT:       return VTERM_VALUETYPE_INT;
     case VTERM_ATTR_FOREGROUND: return VTERM_VALUETYPE_COLOR;
@@ -406,3 +402,20 @@ void vterm_copy_cells(VTermRect dest,
       (*copycell)(pos, srcpos, user);
     }
 }
+
+void vterm_check_version(int major, int minor)
+{
+  if(major != VTERM_VERSION_MAJOR) {
+    fprintf(stderr, "libvterm major version mismatch; %d (wants) != %d (library)\n",
+        major, VTERM_VERSION_MAJOR);
+    exit(1);
+  }
+
+  if(minor > VTERM_VERSION_MINOR) {
+    fprintf(stderr, "libvterm minor version mismatch; %d (wants) > %d (library)\n",
+        minor, VTERM_VERSION_MINOR);
+    exit(1);
+  }
+
+  // Happy
+}

src/libvterm/src/vterm_internal.h

@@ -32,6 +32,9 @@
 #define CSI_ARGS_MAX 16
 #define CSI_LEADER_MAX 16
 
+#define BUFIDX_PRIMARY   0
+#define BUFIDX_ALTSCREEN 1
+
 typedef struct VTermEncoding VTermEncoding;
 
 typedef struct {
@@ -50,6 +53,7 @@ struct VTermPen
   unsigned int italic:1;
   unsigned int blink:1;
   unsigned int reverse:1;
+  unsigned int conceal:1;
   unsigned int strike:1;
   unsigned int font:4; // To store 0-9
 };
@@ -70,7 +74,7 @@ struct VTermState
   const VTermStateCallbacks *callbacks;
   void *cbdata;
 
-  const VTermParserCallbacks *fallbacks;
+  const VTermStateFallbacks *fallbacks;
   void *fbdata;
 
   int rows;
@@ -92,6 +96,10 @@ struct VTermState
   // Bitvector of tab stops
   unsigned char *tabstops;
 
+  /* Primary and Altscreen; lineinfos[1] is lazily allocated as needed */
+  VTermLineInfo *lineinfos[2];
+
+  /* lineinfo will == lineinfos[0] or lineinfos[1], depending on altscreen */
   VTermLineInfo *lineinfo;
 #define ROWWIDTH(state,row) ((state)->lineinfo[(row)].doublewidth ? ((state)->cols / 2) : (state)->cols)
 #define THISROWWIDTH(state) ROWWIDTH(state, (state)->pos.row)
@@ -153,15 +161,13 @@ struct VTermState
       unsigned int cursor_shape:2;
     } mode;
   } saved;
+
+  /* Temporary state for DECRQSS parsing */
+  union {
+    char decrqss[4];
+  } tmp;
 };
 
-typedef enum {
-  VTERM_PARSER_OSC,
-  VTERM_PARSER_DCS,
-
-  VTERM_N_PARSER_TYPES
-} VTermParserStringType;
-
 struct VTerm
 {
   VTermAllocatorFunctions *allocator;
@@ -181,28 +187,39 @@ struct VTerm
       CSI_LEADER,
       CSI_ARGS,
       CSI_INTERMED,
-      ESC,
-      // below here are the "string states"
-      STRING,
-      ESC_IN_STRING,
+      DCS_COMMAND,
+      /* below here are the "string states" */
+      OSC_COMMAND,
+      OSC,
+      DCS,
     } state;
 
+    unsigned int in_esc : 1;
+
     int intermedlen;
     char intermed[INTERMED_MAX];
 
-    int csi_leaderlen;
-    char csi_leader[CSI_LEADER_MAX];
+    union {
+      struct {
+        int leaderlen;
+        char leader[CSI_LEADER_MAX];
 
-    int csi_argi;
-    long csi_args[CSI_ARGS_MAX];
+        int argi;
+        long args[CSI_ARGS_MAX];
+      } csi;
+      struct {
+        int command;
+      } osc;
+      struct {
+        int commandlen;
+        char command[CSI_LEADER_MAX];
+      } dcs;
+    } v;
 
     const VTermParserCallbacks *callbacks;
     void *cbdata;
 
-    VTermParserStringType stringtype;
-    char  *strbuffer;
-    size_t strbuffer_len;
-    size_t strbuffer_cur;
+    int string_initial;
   } parser;
 
   // len == malloc()ed size; cur == number of valid bytes

src/libvterm/t/02parser.test

@@ -132,15 +132,31 @@ PUSH "\e[12\n;3X"
 
 !OSC BEL
 PUSH "\e]1;Hello\x07"
-  osc "1;Hello"
+  osc [1 "Hello"]
 
 !OSC ST (7bit)
 PUSH "\e]1;Hello\e\\"
-  osc "1;Hello"
+  osc [1 "Hello"]
 
 !OSC ST (8bit)
 PUSH "\x{9d}1;Hello\x9c"
-  osc "1;Hello"
+  osc [1 "Hello"]
+
+!OSC in parts
+PUSH "\e]52;abc"
+  osc [52 "abc"
+PUSH "def"
+  osc "def"
+PUSH "ghi\e\\"
+  osc "ghi"]
+
+!OSC BEL without semicolon 
+PUSH "\e]1234\x07"
+  osc [1234 ]
+
+!OSC ST without semicolon 
+PUSH "\e]1234\e\\"
+  osc [1234 ]
 
 !Escape cancels OSC, starts Escape
 PUSH "\e]Something\e9"
@@ -152,20 +168,21 @@ PUSH "\e]12\x{18}AB"
 
 !C0 in OSC interrupts and continues
 PUSH "\e]2;\nBye\x07"
+  osc [2 ""
   control 10
-  osc "2;Bye"
+  osc "Bye"]
 
 !DCS BEL
 PUSH "\ePHello\x07"
-  dcs "Hello"
+  dcs ["Hello"]
 
 !DCS ST (7bit)
 PUSH "\ePHello\e\\"
-  dcs "Hello"
+  dcs ["Hello"]
 
 !DCS ST (8bit)
 PUSH "\x{90}Hello\x9c"
-  dcs "Hello"
+  dcs ["Hello"]
 
 !Escape cancels DCS, starts Escape
 PUSH "\ePSomething\e9"
@@ -177,8 +194,9 @@ PUSH "\eP12\x{18}AB"
 
 !C0 in OSC interrupts and continues
 PUSH "\ePBy\ne\x07"
+  dcs ["By"
   control 10
-  dcs "Bye"
+  dcs "e"]
 
 !NUL ignored
 PUSH "\x{00}"

src/libvterm/t/12state_scroll.test

@@ -40,8 +40,8 @@ PUSH "\n"
   ?cursor = 20,0
 
 !Index in DECSTBM
-PUSH "\e[10H"
 PUSH "\e[9;10r"
+PUSH "\e[10H"
 PUSH "\eM"
   ?cursor = 8,0
 PUSH "\eM"
@@ -148,3 +148,9 @@ PUSH "\e[2T"
   moverect 0..23,0..80 -> 2..25,0..80
   erase 0..2,0..80
   ?cursor = 0,0
+
+!DECSTBM resets cursor position
+PUSH "\e[5;5H"
+  ?cursor = 4,4
+PUSH "\e[r"
+  ?cursor = 0,0

src/libvterm/t/18state_termprops.test

@@ -33,4 +33,10 @@ PUSH "\e[3 q"
 
 !Title
 PUSH "\e]2;Here is my title\a"
-  settermprop 4 "Here is my title"
+  settermprop 4 ["Here is my title"]
+
+!Title split write
+PUSH "\e]2;Here is"
+  settermprop 4 ["Here is"
+PUSH " another title\a"
+  settermprop 4 " another title"]

src/libvterm/t/29state_fallback.test

@@ -12,8 +12,8 @@ PUSH "\e[?15;2z"
 
 !Unrecognised OSC
 PUSH "\e]27;Something\e\\"
-  osc "27;Something"
+  osc [27 "Something"]
 
 !Unrecognised DCS
 PUSH "\ePz123\e\\"
-  dcs "z123"
+  dcs ["z123"]

src/libvterm/t/30state_pen.test

@@ -28,6 +28,14 @@ PUSH "\e[21m"
   ?pen underline = 2
 PUSH "\e[24m"
   ?pen underline = 0
+PUSH "\e[4m\e[4:0m"
+  ?pen underline = 0
+PUSH "\e[4:1m"
+  ?pen underline = 1
+PUSH "\e[4:2m"
+  ?pen underline = 2
+PUSH "\e[4:3m"
+  ?pen underline = 3
 PUSH "\e[4m\e[m"
   ?pen underline = 0
 

src/libvterm/t/32state_flow.test

@@ -0,0 +1,28 @@
+INIT
+WANTSTATE
+
+# Many of these test cases inspired by
+#   https://blueprints.launchpad.net/libvterm/+spec/reflow-cases
+
+!Spillover text marks continuation on second line
+RESET
+PUSH "A"x100
+PUSH "\r\n"
+  ?lineinfo 0 =
+  ?lineinfo 1 = cont
+
+!CRLF in column 80 does not mark continuation
+RESET
+PUSH "B"x80
+PUSH "\r\n"
+PUSH "B"x20
+PUSH "\r\n"
+  ?lineinfo 0 =
+  ?lineinfo 1 =
+
+!EL cancels continuation of following line
+RESET
+PUSH "D"x100
+  ?lineinfo 1 = cont
+PUSH "\eM\e[79G\e[K"
+  ?lineinfo 1 =

src/libvterm/t/60screen_ascii.test

@@ -1,12 +1,12 @@
 INIT
-WANTSCREEN c
+WANTSCREEN ac
 
 !Get
 RESET
 PUSH "ABC"
   movecursor 0,3
-  ?screen_chars 0,0,1,3 = 0x41,0x42,0x43
-  ?screen_chars 0,0,1,80 = 0x41,0x42,0x43
+  ?screen_chars 0,0,1,3 = "ABC"
+  ?screen_chars 0,0,1,80 = "ABC"
   ?screen_text 0,0,1,3 = 0x41,0x42,0x43
   ?screen_text 0,0,1,80 = 0x41,0x42,0x43
   ?screen_cell 0,0 = {0x41} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)
@@ -18,11 +18,11 @@ PUSH "ABC"
   ?screen_eol 0,3 = 1
 PUSH "\e[H"
   movecursor 0,0
-  ?screen_chars 0,0,1,80 = 0x41,0x42,0x43
+  ?screen_chars 0,0,1,80 = "ABC"
   ?screen_text 0,0,1,80 = 0x41,0x42,0x43
 PUSH "E"
   movecursor 0,1
-  ?screen_chars 0,0,1,80 = 0x45,0x42,0x43
+  ?screen_chars 0,0,1,80 = "EBC"
   ?screen_text 0,0,1,80 = 0x45,0x42,0x43
 
 WANTSCREEN -c
@@ -37,33 +37,33 @@ PUSH "ABCDE\e[H\e[K"
 RESET
 PUSH "ABC\e[H\e[@"
 PUSH "1"
-  ?screen_chars 0,0,1,80 = 0x31,0x41,0x42,0x43
+  ?screen_chars 0,0,1,80 = "1ABC"
 
 RESET
 PUSH "ABC\e[H\e[P"
-  ?screen_chars 0,0,1,1 = 0x42
-  ?screen_chars 0,1,1,2 = 0x43
-  ?screen_chars 0,0,1,80 = 0x42,0x43
+  ?screen_chars 0,0,1,1 = "B"
+  ?screen_chars 0,1,1,2 = "C"
+  ?screen_chars 0,0,1,80 = "BC"
 
 !Space padding
 RESET
 PUSH "Hello\e[CWorld"
-  ?screen_chars 0,0,1,80 = 0x48,0x65,0x6c,0x6c,0x6f,0x20,0x57,0x6f,0x72,0x6c,0x64
+  ?screen_chars 0,0,1,80 = "Hello World"
   ?screen_text 0,0,1,80 = 0x48,0x65,0x6c,0x6c,0x6f,0x20,0x57,0x6f,0x72,0x6c,0x64
 
 !Linefeed padding
 RESET
 PUSH "Hello\r\nWorld"
-  ?screen_chars 0,0,2,80 = 0x48,0x65,0x6c,0x6c,0x6f,0x0a,0x57,0x6f,0x72,0x6c,0x64
+  ?screen_chars 0,0,2,80 = "Hello\nWorld"
   ?screen_text 0,0,2,80 = 0x48,0x65,0x6c,0x6c,0x6f,0x0a,0x57,0x6f,0x72,0x6c,0x64
 
 !Altscreen
 RESET
 PUSH "P"
-  ?screen_chars 0,0,1,80 = 0x50
+  ?screen_chars 0,0,1,80 = "P"
 PUSH "\e[?1049h"
   ?screen_chars 0,0,1,80 = 
 PUSH "\e[2K\e[HA"
-  ?screen_chars 0,0,1,80 = 0x41
+  ?screen_chars 0,0,1,80 = "A"
 PUSH "\e[?1049l"
-  ?screen_chars 0,0,1,80 = 0x50
+  ?screen_chars 0,0,1,80 = "P"

src/libvterm/t/62screen_damage.test

@@ -1,5 +1,5 @@
 INIT
-WANTSCREEN Db
+WANTSCREEN aDb
 
 !Putglyph
 RESET
@@ -152,4 +152,4 @@ PUSH "\e[25H\r\nABCDE\b\b\b\e[2P\r\n"
 DAMAGEFLUSH
   moverect 1..25,0..80 -> 0..24,0..80
   damage 24..25,0..80
-  ?screen_chars 23,0,24,5 = 0x41,0x42,0x45
+  ?screen_chars 23,0,24,5 = "ABE"

src/libvterm/t/63screen_resize.test

@@ -6,42 +6,42 @@ WANTSCREEN
 RESET
 RESIZE 25,80
 PUSH "AB\r\nCD"
-  ?screen_chars 0,0,1,80 = 0x41,0x42
-  ?screen_chars 1,0,2,80 = 0x43,0x44
+  ?screen_chars 0,0,1,80 = "AB"
+  ?screen_chars 1,0,2,80 = "CD"
 RESIZE 25,100
-  ?screen_chars 0,0,1,100 = 0x41,0x42
-  ?screen_chars 1,0,2,100 = 0x43,0x44
+  ?screen_chars 0,0,1,100 = "AB"
+  ?screen_chars 1,0,2,100 = "CD"
 
 !Resize wider allows print in new area
 RESET
 RESIZE 25,80
 PUSH "AB\e[79GCD"
-  ?screen_chars 0,0,1,2 = 0x41,0x42
-  ?screen_chars 0,78,1,80 = 0x43,0x44
+  ?screen_chars 0,0,1,2 = "AB"
+  ?screen_chars 0,78,1,80 = "CD"
 RESIZE 25,100
-  ?screen_chars 0,0,1,2 = 0x41,0x42
-  ?screen_chars 0,78,1,80 = 0x43,0x44
+  ?screen_chars 0,0,1,2 = "AB"
+  ?screen_chars 0,78,1,80 = "CD"
 PUSH "E"
-  ?screen_chars 0,78,1,81 = 0x43,0x44,0x45
+  ?screen_chars 0,78,1,81 = "CDE"
 
 !Resize shorter with blanks just truncates
 RESET
 RESIZE 25,80
 PUSH "Top\e[10HLine 10"
-  ?screen_chars 0,0,1,80 = 0x54,0x6f,0x70
-  ?screen_chars 9,0,10,80 = 0x4c,0x69,0x6e,0x65,0x20,0x31,0x30
+  ?screen_chars 0,0,1,80 = "Top"
+  ?screen_chars 9,0,10,80 = "Line 10"
   ?cursor = 9,7
 RESIZE 20,80
-  ?screen_chars 0,0,1,80 = 0x54,0x6f,0x70
-  ?screen_chars 9,0,10,80 = 0x4c,0x69,0x6e,0x65,0x20,0x31,0x30
+  ?screen_chars 0,0,1,80 = "Top"
+  ?screen_chars 9,0,10,80 = "Line 10"
   ?cursor = 9,7
 
 !Resize shorter with content must scroll
 RESET
 RESIZE 25,80
 PUSH "Top\e[25HLine 25\e[15H"
-  ?screen_chars 0,0,1,80 = 0x54,0x6f,0x70
-  ?screen_chars 24,0,25,80 = 0x4c,0x69,0x6e,0x65,0x20,0x32,0x35
+  ?screen_chars 0,0,1,80 = "Top"
+  ?screen_chars 24,0,25,80 = "Line 25"
   ?cursor = 14,0
 WANTSCREEN b
 RESIZE 20,80
@@ -51,7 +51,7 @@ RESIZE 20,80
   sb_pushline 80 =
   sb_pushline 80 =
   ?screen_chars 0,0,1,80 = 
-  ?screen_chars 19,0,20,80 = 0x4c,0x69,0x6e,0x65,0x20,0x32,0x35
+  ?screen_chars 19,0,20,80 = "Line 25"
   ?cursor = 9,0
 
 !Resize shorter does not lose line with cursor
@@ -62,11 +62,11 @@ RESIZE 25,80
 WANTSCREEN b
 PUSH "\e[24HLine 24\r\nLine 25\r\n"
   sb_pushline 80 =
-  ?screen_chars 23,0,24,10 = 0x4c,0x69,0x6e,0x65,0x20,0x32,0x35
+  ?screen_chars 23,0,24,10 = "Line 25"
   ?cursor = 24,0
 RESIZE 24,80
   sb_pushline 80 =
-  ?screen_chars 22,0,23,10 = 0x4c,0x69,0x6e,0x65,0x20,0x32,0x35
+  ?screen_chars 22,0,23,10 = "Line 25"
   ?cursor = 23,0
 
 !Resize taller attempts to pop scrollback
@@ -74,8 +74,8 @@ RESET
 WANTSCREEN -b
 RESIZE 25,80
 PUSH "Line 1\e[25HBottom\e[15H"
-  ?screen_chars 0,0,1,80 = 0x4c,0x69,0x6e,0x65,0x20,0x31
-  ?screen_chars 24,0,25,80 = 0x42,0x6f,0x74,0x74,0x6f,0x6d
+  ?screen_chars 0,0,1,80 = "Line 1"
+  ?screen_chars 24,0,25,80 = "Bottom"
   ?cursor = 14,0
 WANTSCREEN b
 RESIZE 30,80
@@ -84,7 +84,18 @@ RESIZE 30,80
   sb_popline 80
   sb_popline 80
   sb_popline 80
-  ?screen_chars 0,0,1,80 = 0x41,0x42,0x43,0x44,0x45
-  ?screen_chars 5,0,6,80 = 0x4c,0x69,0x6e,0x65,0x20,0x31
-  ?screen_chars 29,0,30,80 = 0x42,0x6f,0x74,0x74,0x6f,0x6d
+  ?screen_chars 0,0,1,80 = "ABCDE"
+  ?screen_chars 5,0,6,80 = "Line 1"
+  ?screen_chars 29,0,30,80 = "Bottom"
   ?cursor = 19,0
+WANTSCREEN -b
+
+!Resize can operate on altscreen
+RESET
+WANTSCREEN a
+RESIZE 25,80
+PUSH "Main screen\e[?1049h\e[HAlt screen"
+RESIZE 30,80
+  ?screen_chars 0,0,1,3 = "Alt"
+PUSH "\e[?1049l"
+  ?screen_chars 0,0,1,3 = "Mai"

src/libvterm/t/67screen_dbl_wh.test

@@ -30,3 +30,9 @@ PUSH "abcde"
   ?screen_cell 0,0 = {0x61} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)
 PUSH "\e#6"
   ?screen_cell 0,0 = {0x61} width=1 attrs={} dwl fg=rgb(240,240,240) bg=rgb(0,0,0)
+
+!DWL doesn't spill over on scroll
+RESET
+PUSH "\e[25H\e#6Final\r\n"
+  ?screen_cell 23,0 = {0x46} width=1 attrs={} dwl fg=rgb(240,240,240) bg=rgb(0,0,0)
+  ?screen_cell 24,0 = {} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)

src/libvterm/t/68screen_termprops.test

@@ -14,4 +14,4 @@ PUSH "\e[?25l"
 
 !Title
 PUSH "\e]2;Here is my title\a"
-  settermprop 4 "Here is my title"
+  settermprop 4 ["Here is my title"]

src/libvterm/t/harness.c

@@ -153,21 +153,44 @@ static int parser_csi(const char *leader, const long args[], int argcount, const
   return 1;
 }
 
-static int parser_osc(const char *command, size_t cmdlen, void *user UNUSED)
+static int parser_osc(int command, VTermStringFragment frag, void *user UNUSED)
 {
 
   printf("osc ");
-  printhex(command, cmdlen);
+
+  if(frag.initial) {
+    if(command == -1)
+      printf("[");
+    else
+      printf("[%d;", command);
+  }
+
+  printhex(frag.str, frag.len);
+
+  if(frag.final)
+    printf("]");
+
   printf("\n");
 
   return 1;
 }
 
-static int parser_dcs(const char *command, size_t cmdlen, void *user UNUSED)
+static int parser_dcs(const char *command, size_t commandlen, VTermStringFragment frag, void *user UNUSED)
 {
-
   printf("dcs ");
-  printhex(command, cmdlen);
+
+  if(frag.initial) {
+    size_t i;
+    printf("[");
+    for(i = 0; i < commandlen; i++)
+      printf("%02x", command[i]);
+  }
+
+  printhex(frag.str, frag.len);
+
+  if(frag.final)
+    printf("]");
+
   printf("\n");
 
   return 1;
@@ -183,7 +206,14 @@ static VTermParserCallbacks parser_cbs = {
   NULL // resize
 };
 
-// These callbacks are shared by State and Screen
+static VTermStateFallbacks fallbacks = {
+  parser_control, // control
+  parser_csi, // csi
+  parser_osc, // osc
+  parser_dcs // dcs
+};
+
+/* These callbacks are shared by State and Screen */
 
 static int want_movecursor = 0;
 static VTermPos state_pos;
@@ -239,7 +269,8 @@ static int settermprop(VTermProp prop, VTermValue *val, void *user UNUSED)
     printf("settermprop %d %d\n", prop, val->number);
     return 1;
   case VTERM_VALUETYPE_STRING:
-    printf("settermprop %d \"%s\"\n", prop, val->string);
+    printf("settermprop %d %s\"%.*s\"%s\n", prop,
+        val->string.initial ? "[" : "", val->string.len, val->string.str, val->string.final ? "]" : "");
     return 1;
   case VTERM_VALUETYPE_COLOR:
     printf("settermprop %d rgb(%d,%d,%d)\n", prop, val->color.red, val->color.green, val->color.blue);
@@ -262,7 +293,7 @@ static int state_putglyph(VTermGlyphInfo *info, VTermPos pos, void *user UNUSED)
     return 1;
 
   printf("putglyph ");
-  for(i = 0; info->chars[i]; i++)
+  for(i = 0; i < VTERM_MAX_CHARS_PER_CELL && info->chars[i]; i++)
     printf(i ? ",%x" : "%x", info->chars[i]);
   printf(" %d %d,%d", info->width, pos.row, pos.col);
   if(info->protected_cell)
@@ -295,6 +326,7 @@ static struct {
   int italic;
   int blink;
   int reverse;
+  int conceal;
   int strike;
   int font;
   VTermColor foreground;
@@ -318,6 +350,9 @@ static int state_setpenattr(VTermAttr attr, VTermValue *val, void *user UNUSED)
   case VTERM_ATTR_REVERSE:
     state_pen.reverse = val->boolean;
     break;
+  case VTERM_ATTR_CONCEAL:
+    state_pen.conceal = val->boolean;
+    break;
   case VTERM_ATTR_STRIKE:
     state_pen.strike = val->boolean;
     break;
@@ -528,7 +563,7 @@ int main(int argc UNUSED, char **argv UNUSED)
           want_settermprop = sense;
           break;
         case 'f':
-          vterm_state_set_unrecognised_fallbacks(state, sense ? &parser_cbs : NULL, NULL);
+          vterm_state_set_unrecognised_fallbacks(state, sense ? &fallbacks : NULL, NULL);
           break;
         default:
           fprintf(stderr, "Unrecognised WANTSTATE flag '%c'\n", line[i]);
@@ -540,7 +575,6 @@ int main(int argc UNUSED, char **argv UNUSED)
       int sense = 1;
       if(!screen)
         screen = vterm_obtain_screen(vt);
-      vterm_screen_enable_altscreen(screen, 1);
       vterm_screen_set_callbacks(screen, &screen_cbs, NULL);
 
       while(line[i] == ' ')
@@ -550,6 +584,9 @@ int main(int argc UNUSED, char **argv UNUSED)
         case '-':
           sense = 0;
           break;
+        case 'a':
+          vterm_screen_enable_altscreen(screen, 1);
+          break;
         case 'd':
           want_screen_damage = sense;
           break;
@@ -805,6 +842,25 @@ int main(int argc UNUSED, char **argv UNUSED)
         else
           printf("?\n");
       }
+      else if(strstartswith(line, "?lineinfo ")) {
+        char *linep = line + 10;
+        int row;
+        const VTermLineInfo *info;
+        while(linep[0] == ' ')
+          linep++;
+        if(sscanf(linep, "%d", &row) < 1) {
+          printf("! lineinfo unrecognised input\n");
+          goto abort_line;
+        }
+        info = vterm_state_get_lineinfo(state, row);
+        if(info->doublewidth)
+          printf("dwl ");
+        if(info->doubleheight)
+          printf("dhl ");
+        if(info->continuation)
+          printf("cont ");
+        printf("\n");
+      }
       else if(strstartswith(line, "?screen_chars ")) {
         char *linep = line + 13;
         VTermRect rect;

src/libvterm/t/run-test.pl

@@ -11,7 +11,8 @@ my $VALGRIND = 0;
 my $EXECUTABLE = "t/.libs/harness";
 GetOptions(
    'valgrind|v+' => \$VALGRIND,
-   'executable|e=s' => \$EXECUTABLE
+   'executable|e=s' => \$EXECUTABLE,
+   'fail-early|F' => \(my $FAIL_EARLY),
 ) or exit 1;
 
 my ( $hin, $hout, $hpid );
@@ -28,6 +29,8 @@ my $exitcode = 0;
 my $command;
 my @expect;
 
+my $linenum = 0;
+
 sub do_onetest
 {
    $hin->print( "$command\n" );
@@ -41,7 +44,7 @@ sub do_onetest
       chomp $outline;
 
       if( !@expect ) {
-         print "# Test failed\n" unless $fail_printed++;
+         print "# line $linenum: Test failed\n" unless $fail_printed++;
          print "#    expected nothing more\n" .
                "#   Actual:   $outline\n";
          next;
@@ -51,18 +54,19 @@ sub do_onetest
 
       next if $expectation eq $outline;
 
-      print "# Test failed\n" unless $fail_printed++;
+      print "# line $linenum: Test failed\n" unless $fail_printed++;
       print "#   Expected: $expectation\n" .
             "#   Actual:   $outline\n";
    }
 
    if( @expect ) {
-      print "# Test failed\n" unless $fail_printed++;
+      print "# line $linenum: Test failed\n" unless $fail_printed++;
       print "#   Expected: $_\n" .
             "#    didn't happen\n" for @expect;
    }
 
    $exitcode = 1 if $fail_printed;
+   exit $exitcode if $exitcode and $FAIL_EARLY;
 }
 
 sub do_line
@@ -103,8 +107,15 @@ sub do_line
       elsif( $line =~ m/^csi (\S+) (.*)$/ ) {
          $line = sprintf "csi %02x %s", eval($1), $2; # TODO
       }
-      elsif( $line =~ m/^(escape|osc|dcs) (.*)$/ ) {
-         $line = "$1 " . join "", map sprintf("%02x", $_), unpack "C*", eval($2);
+      elsif( $line =~ m/^(osc) (\[\d+)? *(.*?)(\]?)$/ ) {
+         my ( $cmd, $initial, $data, $final ) = ( $1, $2, $3, $4 );
+         $initial //= "";
+         $initial .= ";" if $initial =~ m/\d+/;
+
+         $line = "$cmd $initial" . join( "", map sprintf("%02x", $_), unpack "C*", length $data ? eval($data) : "" ) . "$final";
+      }
+      elsif( $line =~ m/^(escape|dcs) (\[?)(.*?)(\]?)$/ ) {
+         $line = "$1 $2" . join( "", map sprintf("%02x", $_), unpack "C*", eval($3) ) . "$4";
       }
       elsif( $line =~ m/^putglyph (\S+) (.*)$/ ) {
          $line = "putglyph " . join( ",", map sprintf("%x", $_), eval($1) ) . " $2";
@@ -133,27 +144,35 @@ sub do_line
 
       $response = pack "C*", map hex, split m/,/, $response;
       if( $response ne $want ) {
-         print "# Assert ?screen_row $row failed:\n" .
+         print "# line $linenum: Assert ?screen_row $row failed:\n" .
                "# Expected: $want\n" .
                "# Actual:   $response\n";
          $exitcode = 1;
+         exit $exitcode if $exitcode and $FAIL_EARLY;
       }
    }
    # Assertions start with '?'
-   elsif( $line =~ s/^\?([a-z]+.*?=)\s+// ) {
+   elsif( $line =~ s/^\?([a-z]+.*?=)\s*// ) {
       do_onetest if defined $command;
 
       my ( $assertion ) = $1 =~ m/^(.*)\s+=/;
+      my $expectation = $line;
 
       $hin->print( "\?$assertion\n" );
       my $response = <$hout>; defined $response or wait, die "Test harness failed - $?\n";
-      chomp $response;
+      chomp $response; $response =~ s/^\s+|\s+$//g;
 
-      if( $response ne $line ) {
-         print "# Assert $assertion failed:\n" .
-               "# Expected: $line\n" .
+      # Some convenience formatting
+      if( $assertion =~ m/^screen_chars/ and $expectation =~ m/^"/ ) {
+         $expectation = join ",", map sprintf("0x%02x", ord $_), split m//, eval($expectation);
+      }
+
+      if( $response ne $expectation ) {
+         print "# line $linenum: Assert $assertion failed:\n" .
+               "# Expected: $expectation\n" .
                "# Actual:   $response\n";
          $exitcode = 1;
+         exit $exitcode if $exitcode and $FAIL_EARLY;
       }
    }
    # Test controls start with '$'
@@ -176,10 +195,13 @@ sub do_line
 open my $test, "<", $ARGV[0] or die "Cannot open test script $ARGV[0] - $!";
 
 while( my $line = <$test> ) {
+   $linenum++;
    $line =~ s/^\s+//;
-   next if $line =~ m/^(?:#|$)/;
-
    chomp $line;
+
+   next if $line =~ m/^(?:#|$)/;
+   last if $line eq "__END__";
+
    do_line( $line );
 }
 

src/os_win32.c

@@ -822,7 +822,7 @@ win32_enable_privilege(LPTSTR lpszPrivilege, BOOL bEnable)
 #endif
 
 /*
- * Set "win8_or_later" and fill in "windowsVersion".
+ * Set "win8_or_later" and fill in "windowsVersion" if possible.
  */
     void
 PlatformId(void)
@@ -836,9 +836,10 @@ PlatformId(void)
 	ovi.dwOSVersionInfoSize = sizeof(ovi);
 	GetVersionEx(&ovi);
 
+#ifdef FEAT_EVAL
 	vim_snprintf(windowsVersion, sizeof(windowsVersion), "%d.%d",
 		(int)ovi.dwMajorVersion, (int)ovi.dwMinorVersion);
-
+#endif
 	if ((ovi.dwMajorVersion == 6 && ovi.dwMinorVersion >= 2)
 		|| ovi.dwMajorVersion > 6)
 	    win8_or_later = TRUE;

src/popupwin.c

@@ -1757,6 +1757,27 @@ add_border_left_right_padding(win_T *wp)
     }
 }
 
+#ifdef FEAT_TERMINAL
+/*
+ * Return TRUE if there is any popup window with a terminal buffer.
+ */
+    static int
+popup_terminal_exists(void)
+{
+    win_T	*wp;
+    tabpage_T	*tp;
+
+    FOR_ALL_POPUPWINS(wp)
+	if (wp->w_buffer->b_term != NULL)
+	    return TRUE;
+    FOR_ALL_TABPAGES(tp)
+	FOR_ALL_POPUPWINS_IN_TAB(tp, wp)
+	    if (wp->w_buffer->b_term != NULL)
+		return TRUE;
+    return FALSE;
+}
+#endif
+
 /*
  * popup_create({text}, {options})
  * popup_atcursor({text}, {options})
@@ -1786,6 +1807,13 @@ popup_create(typval_T *argvars, typval_T *rettv, create_type_T type)
 		semsg(_(e_nobufnr), argvars[0].vval.v_number);
 		return NULL;
 	    }
+#ifdef FEAT_TERMINAL
+	    if (buf->b_term != NULL && popup_terminal_exists())
+	    {
+		emsg(_("E861: Cannot open a second popup with a terminal"));
+		return NULL;
+	    }
+#endif
 	}
 	else if (!(argvars[0].v_type == VAR_STRING
 			&& argvars[0].vval.v_string != NULL)

src/terminal.c

@@ -162,6 +162,7 @@ struct terminal_S {
     char_u	*tl_cursor_color; // NULL or allocated
 
     int		tl_using_altscreen;
+    garray_T	tl_osc_buf;	    // incomplete OSC string
 };
 
 #define TMODE_ONCE 1	    // CTRL-\ CTRL-N used
@@ -445,6 +446,7 @@ term_start(
 #endif
     ga_init2(&term->tl_scrollback, sizeof(sb_line_T), 300);
     ga_init2(&term->tl_scrollback_postponed, sizeof(sb_line_T), 300);
+    ga_init2(&term->tl_osc_buf, sizeof(char), 300);
 
     CLEAR_FIELD(split_ea);
     if (opt->jo_curwin)
@@ -1015,6 +1017,7 @@ free_unused_terminals()
 	terminals_to_free = term->tl_next;
 
 	free_scrollback(term);
+	ga_clear(&term->tl_osc_buf);
 
 	term_free_vterm(term);
 	vim_free(term->tl_api);
@@ -2998,22 +3001,27 @@ handle_settermprop(
 	void *user)
 {
     term_T	*term = (term_T *)user;
+    char_u	*strval = NULL;
 
     switch (prop)
     {
 	case VTERM_PROP_TITLE:
+	    strval = vim_strnsave((char_u *)value->string.str,
+						       (int)value->string.len);
+	    if (strval == NULL)
+		break;
 	    vim_free(term->tl_title);
 	    // a blank title isn't useful, make it empty, so that "running" is
 	    // displayed
-	    if (*skipwhite((char_u *)value->string) == NUL)
+	    if (*skipwhite(strval) == NUL)
 		term->tl_title = NULL;
 	    // Same as blank
 	    else if (term->tl_arg0_cmd != NULL
-		    && STRNCMP(term->tl_arg0_cmd, (char_u *)value->string,
+		    && STRNCMP(term->tl_arg0_cmd, strval,
 					  (int)STRLEN(term->tl_arg0_cmd)) == 0)
 		term->tl_title = NULL;
 	    // Empty corrupted data of winpty
-	    else if (STRNCMP("  - ", (char_u *)value->string, 4) == 0)
+	    else if (STRNCMP("  - ", strval, 4) == 0)
 		term->tl_title = NULL;
 #ifdef MSWIN
 	    else if (!enc_utf8 && enc_codepage > 0)
@@ -3022,8 +3030,8 @@ handle_settermprop(
 		int	length = 0;
 
 		MultiByteToWideChar_alloc(CP_UTF8, 0,
-			(char*)value->string, (int)STRLEN(value->string),
-								&ret, &length);
+			(char*)value->string.str,
+					(int)value->string.len, &ret, &length);
 		if (ret != NULL)
 		{
 		    WideCharToMultiByte_alloc(enc_codepage, 0,
@@ -3034,7 +3042,10 @@ handle_settermprop(
 	    }
 #endif
 	    else
-		term->tl_title = vim_strsave((char_u *)value->string);
+	    {
+		term->tl_title = vim_strsave(strval);
+		strval = NULL;
+	    }
 	    VIM_CLEAR(term->tl_status_text);
 	    if (term == curbuf->b_term)
 		maketitle();
@@ -3057,7 +3068,11 @@ handle_settermprop(
 	    break;
 
 	case VTERM_PROP_CURSORCOLOR:
-	    cursor_color_copy(&term->tl_cursor_color, (char_u*)value->string);
+	    strval = vim_strnsave((char_u *)value->string.str,
+						       (int)value->string.len);
+	    if (strval == NULL)
+		break;
+	    cursor_color_copy(&term->tl_cursor_color, strval);
 	    may_set_cursor_props(term);
 	    break;
 
@@ -3069,6 +3084,8 @@ handle_settermprop(
 	default:
 	    break;
     }
+    vim_free(strval);
+
     // Always return 1, otherwise vterm doesn't store the value internally.
     return 1;
 }
@@ -4181,21 +4198,32 @@ handle_call_command(term_T *term, channel_T *channel, listitem_T *item)
  * We recognize a terminal API command.
  */
     static int
-parse_osc(const char *command, size_t cmdlen, void *user)
+parse_osc(int command, VTermStringFragment frag, void *user)
 {
     term_T	*term = (term_T *)user;
     js_read_T	reader;
     typval_T	tv;
     channel_T	*channel = term->tl_job == NULL ? NULL
 						    : term->tl_job->jv_channel;
+    garray_T	*gap = &term->tl_osc_buf;
 
     // We recognize only OSC 5 1 ; {command}
-    if (cmdlen < 3 || STRNCMP(command, "51;", 3) != 0)
-	return 0; // not handled
+    if (command != 51)
+	return 0;
 
-    reader.js_buf = vim_strnsave((char_u *)command + 3, (int)(cmdlen - 3));
-    if (reader.js_buf == NULL)
+    // Concatenate what was received until the final piece is found.
+    if (ga_grow(gap, (int)frag.len + 1) == FAIL)
+    {
+	ga_clear(gap);
 	return 1;
+    }
+    mch_memmove((char *)gap->ga_data + gap->ga_len, frag.str, frag.len);
+    gap->ga_len += frag.len;
+    if (!frag.final)
+	return 1;
+
+    ((char *)gap->ga_data)[gap->ga_len] = 0;
+    reader.js_buf = gap->ga_data;
     reader.js_fill = NULL;
     reader.js_used = 0;
     if (json_decode(&reader, &tv, 0) == OK
@@ -4229,7 +4257,7 @@ parse_osc(const char *command, size_t cmdlen, void *user)
     else
 	ch_log(channel, "Invalid JSON received");
 
-    vim_free(reader.js_buf);
+    ga_clear(gap);
     clear_tv(&tv);
     return 1;
 }
@@ -4293,14 +4321,11 @@ parse_csi(
     return 1;
 }
 
-static VTermParserCallbacks parser_fallbacks = {
-  NULL,		// text
+static VTermStateFallbacks state_fallbacks = {
   NULL,		// control
-  NULL,		// escape
   parse_csi,	// csi
   parse_osc,	// osc
-  NULL,		// dcs
-  NULL		// resize
+  NULL		// dcs
 };
 
 /*
@@ -4383,7 +4408,7 @@ create_vterm(term_T *term, int rows, int cols)
     value.boolean = 0;
 #endif
     vterm_state_set_termprop(state, VTERM_PROP_CURSORBLINK, &value);
-    vterm_state_set_unrecognised_fallbacks(state, &parser_fallbacks, term);
+    vterm_state_set_unrecognised_fallbacks(state, &state_fallbacks, term);
 
     return OK;
 }

src/testdir/test_channel.py

@@ -21,6 +21,9 @@ except ImportError:
 
 class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
 
+    def setup(self):
+        self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
     def handle(self):
         print("=== socket opened ===")
         while True:

src/testdir/test_json.vim

@@ -148,36 +148,36 @@ func Test_json_decode()
   call assert_fails("call json_decode('{\"\": \"ok\", \"\": \"bad\"}')", 'E938:')
 
   call assert_equal({'n': 1}, json_decode('{"n":1,}'))
-  call assert_fails("call json_decode(\"{'n':'1',}\")", 'E474:')
-  call assert_fails("call json_decode(\"'n'\")", 'E474:')
+  call assert_fails("call json_decode(\"{'n':'1',}\")", 'E491:')
+  call assert_fails("call json_decode(\"'n'\")", 'E491:')
 
-  call assert_fails('call json_decode("\"")', "E474:")
-  call assert_fails('call json_decode("blah")', "E474:")
+  call assert_fails('call json_decode("\"")', "E491:")
+  call assert_fails('call json_decode("blah")', "E491:")
   call assert_fails('call json_decode("true blah")', "E488:")
-  call assert_fails('call json_decode("<foobar>")', "E474:")
+  call assert_fails('call json_decode("<foobar>")', "E491:")
   call assert_fails('call json_decode("{\"a\":1,\"a\":2}")', "E938:")
 
-  call assert_fails('call json_decode("{")', "E474:")
-  call assert_fails('call json_decode("{foobar}")', "E474:")
-  call assert_fails('call json_decode("{\"n\",")', "E474:")
-  call assert_fails('call json_decode("{\"n\":")', "E474:")
-  call assert_fails('call json_decode("{\"n\":1")', "E474:")
-  call assert_fails('call json_decode("{\"n\":1,")', "E474:")
-  call assert_fails('call json_decode("{\"n\",1}")', "E474:")
-  call assert_fails('call json_decode("{-}")', "E474:")
+  call assert_fails('call json_decode("{")', "E491:")
+  call assert_fails('call json_decode("{foobar}")', "E491:")
+  call assert_fails('call json_decode("{\"n\",")', "E491:")
+  call assert_fails('call json_decode("{\"n\":")', "E491:")
+  call assert_fails('call json_decode("{\"n\":1")', "E491:")
+  call assert_fails('call json_decode("{\"n\":1,")', "E491:")
+  call assert_fails('call json_decode("{\"n\",1}")', "E491:")
+  call assert_fails('call json_decode("{-}")', "E491:")
 
-  call assert_fails('call json_decode("[foobar]")', "E474:")
-  call assert_fails('call json_decode("[")', "E474:")
-  call assert_fails('call json_decode("[1")', "E474:")
-  call assert_fails('call json_decode("[1,")', "E474:")
-  call assert_fails('call json_decode("[1 2]")', "E474:")
+  call assert_fails('call json_decode("[foobar]")', "E491:")
+  call assert_fails('call json_decode("[")', "E491:")
+  call assert_fails('call json_decode("[1")', "E491:")
+  call assert_fails('call json_decode("[1,")', "E491:")
+  call assert_fails('call json_decode("[1 2]")', "E491:")
 
-  call assert_fails('call json_decode("[1,,2]")', "E474:")
+  call assert_fails('call json_decode("[1,,2]")', "E491:")
 
-  call assert_fails('call json_decode("{{}:42}")', "E474:")
-  call assert_fails('call json_decode("{[]:42}")', "E474:")
+  call assert_fails('call json_decode("{{}:42}")', "E491:")
+  call assert_fails('call json_decode("{[]:42}")', "E491:")
 
-  call assert_fails('call json_decode("\"\\u111Z\"")', 'E474:')
+  call assert_fails('call json_decode("\"\\u111Z\"")', 'E491:')
   call assert_equal('[😂]', json_decode('"[\uD83D\uDE02]"'))
   call assert_equal('a😂b', json_decode('"a\uD83D\uDE02b"'))
 endfunc

src/testdir/test_netbeans.vim

@@ -609,7 +609,7 @@ func Nb_basic(port)
 
   " detach
   call appendbufline(cmdbufnr, '$', 'detach_Test')
-  call WaitFor('len(readfile("Xnetbeans")) >= (g:last + 6)')
+  call WaitFor('len(readfile("Xnetbeans")) >= (g:last + 8)')
   call WaitForAssert({-> assert_equal('0:disconnect=91', readfile("Xnetbeans")[-1])})
 
   " the connection was closed

src/testdir/test_popupwin.vim

@@ -2426,10 +2426,10 @@ func Test_popupwin_terminal_buffer()
   let g:test_is_flaky = 1
 
   let origwin = win_getid()
-  let ptybuf = term_start(&shell, #{hidden: 1})
-  let winid = popup_create(ptybuf, #{minwidth: 40, minheight: 10})
+  let termbuf = term_start(&shell, #{hidden: 1})
+  let winid = popup_create(termbuf, #{minwidth: 40, minheight: 10})
   " Wait for shell to start
-  call WaitForAssert({-> assert_equal("run", job_status(term_getjob(ptybuf)))})
+  call WaitForAssert({-> assert_equal("run", job_status(term_getjob(termbuf)))})
   sleep 100m
   " Check this doesn't crash
   call assert_equal(winnr(), winnr('j'))
@@ -2440,11 +2440,16 @@ func Test_popupwin_terminal_buffer()
   " Cannot quit while job is running
   call assert_fails('call feedkeys("\<C-W>:quit\<CR>", "xt")', 'E948:')
 
-  " Cannot enter Terminal-Normal mode.
+  " Cannot enter Terminal-Normal mode. (TODO: but it works...)
   call feedkeys("xxx\<C-W>N", 'xt')
   call assert_fails('call feedkeys("gf", "xt")', 'E863:')
   call feedkeys("a\<C-U>", 'xt')
 
+  " Cannot open a second one.
+  let termbuf2 = term_start(&shell, #{hidden: 1})
+  call assert_fails('call popup_create(termbuf2, #{})', 'E861:')
+  call term_sendkeys(termbuf2, "exit\<CR>")
+
   " Exiting shell closes popup window
   call feedkeys("exit\<CR>", 'xt')
   " Wait for shell to exit

src/testdir/test_terminal.vim

@@ -2587,9 +2587,8 @@ func Test_double_popup_terminal()
   let buf1 = term_start(&shell, #{hidden: 1})
   let win1 = popup_create(buf1, {})
   let buf2 = term_start(&shell, #{hidden: 1})
-  let win2 = popup_create(buf2, {})
+  call assert_fails('call popup_create(buf2, {})', 'E861:')
   call popup_close(win1)
-  call popup_close(win2)
   exe buf1 .. 'bwipe!'
   exe buf2 .. 'bwipe!'
 endfunc
@@ -2619,10 +2618,8 @@ func Test_term_nasty_callback()
   CheckExecutable sh
 
   set hidden
-  let g:buf0 = term_start('sh', #{hidden: 1})
+  let g:buf0 = term_start('sh', #{hidden: 1, term_finish: 'close'})
   call popup_create(g:buf0, {})
-  let g:buf1 = term_start('sh', #{hidden: 1, term_finish: 'close'})
-  call popup_create(g:buf1, {})
   call assert_fails("call term_start(['sh', '-c'], #{curwin: 1})", 'E863:')
 
   call popup_clear(1)

src/testdir/test_vim9_expr.vim

@@ -153,9 +153,13 @@ let adict = #{aaa: 2, bbb: 8}
 
 " test == comperator
 def Test_expr4_equal()
+  let trueVar = true
+  let falseVar = false
   assert_equal(true, true == true)
   assert_equal(false, true ==
 			false)
+  assert_equal(true, true == trueVar)
+  assert_equal(false, true == falseVar)
   assert_equal(true, true == g:atrue)
   assert_equal(false, g:atrue == false)
 
@@ -164,8 +168,12 @@ def Test_expr4_equal()
   assert_equal(true, g:anone == v:none)
   assert_equal(false, v:none == g:anull)
 
+  let nr0 = 0
+  let nr61 = 61
   assert_equal(false, 2 == 0)
+  assert_equal(false, 2 == nr0)
   assert_equal(true, 61 == 61)
+  assert_equal(true, 61 == nr61)
   assert_equal(true, g:anint == 10)
   assert_equal(false, 61 == g:anint)
 
@@ -237,9 +245,13 @@ enddef
 
 " test != comperator
 def Test_expr4_notequal()
+  let trueVar = true
+  let falseVar = false
   assert_equal(false, true != true)
   assert_equal(true, true !=
 			false)
+  assert_equal(false, true != trueVar)
+  assert_equal(true, true != falseVar)
   assert_equal(false, true != g:atrue)
   assert_equal(true, g:atrue != false)
 
@@ -248,8 +260,12 @@ def Test_expr4_notequal()
   assert_equal(false, g:anone != v:none)
   assert_equal(true, v:none != g:anull)
 
+  let nr55 = 55
+  let nr0 = 55
   assert_equal(true, 2 != 0)
+  assert_equal(true, 2 != nr0)
   assert_equal(false, 55 != 55)
+  assert_equal(false, 55 != nr55)
   assert_equal(false, g:anint != 10)
   assert_equal(true, 61 != g:anint)
 
@@ -313,6 +329,12 @@ def Test_expr4_greater()
 		1)
   assert_false(2 > 2)
   assert_false(2 > 3)
+  let nr2 = 2
+  assert_true(nr2 > 0)
+  assert_true(nr2 >
+		1)
+  assert_false(nr2 > 2)
+  assert_false(nr2 > 3)
   if has('float')
     let ff = 2.0
     assert_true(ff > 0.0)
@@ -328,6 +350,10 @@ def Test_expr4_greaterequal()
   assert_true(2 >=
 			2)
   assert_false(2 >= 3)
+  let nr2 = 2
+  assert_true(nr2 >= 0)
+  assert_true(nr2 >= 2)
+  assert_false(nr2 >= 3)
   if has('float')
     let ff = 2.0
     assert_true(ff >= 0.0)
@@ -342,6 +368,10 @@ def Test_expr4_smaller()
   assert_false(2 <
 			2)
   assert_true(2 < 3)
+  let nr2 = 2
+  assert_false(nr2 < 0)
+  assert_false(nr2 < 2)
+  assert_true(nr2 < 3)
   if has('float')
     let ff = 2.0
     assert_false(ff < 0.0)
@@ -357,6 +387,11 @@ def Test_expr4_smallerequal()
 			1)
   assert_true(2 <= 2)
   assert_true(2 <= 3)
+  let nr2 = 2
+  assert_false(nr2 <= 0)
+  assert_false(nr2 <= 1)
+  assert_true(nr2 <= 2)
+  assert_true(nr2 <= 3)
   if has('float')
     let ff = 2.0
     assert_false(ff <= 0.0)

src/testdir/test_vim9_script.vim

@@ -23,21 +23,6 @@ def Test_assignment()
   let bool2: bool = false
   assert_equal(v:false, bool2)
 
-  let list1: list<bool> = [false, true, false]
-  let list2: list<number> = [1, 2, 3]
-  let list3: list<string> = ['sdf', 'asdf']
-  let list4: list<any> = ['yes', true, 1234]
-  let list5: list<blob> = [0z01, 0z02]
-
-  let listS: list<string> = []
-  let listN: list<number> = []
-
-  let dict1: dict<bool> = #{one: false, two: true}
-  let dict2: dict<number> = #{one: 1, two: 2}
-  let dict3: dict<string> = #{key: 'value'}
-  let dict4: dict<any> = #{one: 1, two: '2'}
-  let dict5: dict<blob> = #{one: 0z01, tw: 0z02}
-
   call CheckDefFailure(['let x:string'], 'E1069:')
   call CheckDefFailure(['let x:string = "x"'], 'E1069:')
   call CheckDefFailure(['let a:string = "x"'], 'E1069:')
@@ -57,11 +42,6 @@ def Test_assignment()
   let Funky2: func = function('len')
   let Party2: func = funcref('g:Test_syntax')
 
-  # type becomes list<any>
-  let somelist = rand() > 0 ? [1, 2, 3] : ['a', 'b', 'c']
-  # type becomes dict<any>
-  let somedict = rand() > 0 ? #{a: 1, b: 2} : #{a: 'a', b: 'b'}
-
   g:newvar = 'new'
   assert_equal('new', g:newvar)
 
@@ -128,6 +108,42 @@ def Test_assignment()
   call CheckDefFailure(['v:errmsg += 123'], 'E1013:')
 enddef
 
+def Test_assignment_list()
+  let list1: list<bool> = [false, true, false]
+  let list2: list<number> = [1, 2, 3]
+  let list3: list<string> = ['sdf', 'asdf']
+  let list4: list<any> = ['yes', true, 1234]
+  let list5: list<blob> = [0z01, 0z02]
+
+  let listS: list<string> = []
+  let listN: list<number> = []
+
+  assert_equal([1, 2, 3], list2)
+  list2[-1] = 99
+  assert_equal([1, 2, 99], list2)
+  list2[-2] = 88
+  assert_equal([1, 88, 99], list2)
+  list2[-3] = 77
+  assert_equal([77, 88, 99], list2)
+  call CheckDefExecFailure(['let ll = [1, 2, 3]', 'll[-4] = 6'], 'E684:')
+
+  # type becomes list<any>
+  let somelist = rand() > 0 ? [1, 2, 3] : ['a', 'b', 'c']
+enddef
+
+def Test_assignment_dict()
+  let dict1: dict<bool> = #{one: false, two: true}
+  let dict2: dict<number> = #{one: 1, two: 2}
+  let dict3: dict<string> = #{key: 'value'}
+  let dict4: dict<any> = #{one: 1, two: '2'}
+  let dict5: dict<blob> = #{one: 0z01, tw: 0z02}
+
+  call CheckDefExecFailure(['let dd = {}', 'dd[""] = 6'], 'E713:')
+
+  # type becomes dict<any>
+  let somedict = rand() > 0 ? #{a: 1, b: 2} : #{a: 'a', b: 'b'}
+enddef
+
 def Test_assignment_local()
   " Test in a separated file in order not to the current buffer/window/tab is
   " changed.

src/version.c

@@ -746,6 +746,44 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    803,
+/**/
+    802,
+/**/
+    801,
+/**/
+    800,
+/**/
+    799,
+/**/
+    798,
+/**/
+    797,
+/**/
+    796,
+/**/
+    795,
+/**/
+    794,
+/**/
+    793,
+/**/
+    792,
+/**/
+    791,
+/**/
+    790,
+/**/
+    789,
+/**/
+    788,
+/**/
+    787,
+/**/
+    786,
+/**/
+    785,
 /**/
     784,
 /**/