patch 8.2.0804: libvterm code lags behind the upstream version

Problem:    Libvterm code lags behind the upstream version.
Solution:   Include revision 727, but add the index instead of switching
            between RGB and indexed.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -8,7 +8,7 @@ labels: enhancement
 
 _Instructions: Replace the template text and remove irrelevant text (including this line)_
 
-**Is your feature request related something that is currently hard to do? Please describe.**
+**Is your feature request about something that is currently impossible or hard to do? Please describe the problem.**
 A clear and concise description of what is hard to do. Ex. It is difficult to [...] when [...]
 (If it is related to runtime files, please check their header for where to discuss enhancements.)
 

Filelist

@@ -301,10 +301,12 @@ SRC_ALL =	\
 		src/libvterm/LICENSE \
 		src/libvterm/Makefile \
 		src/libvterm/README \
+		src/libvterm/CONTRIBUTING \
 		src/libvterm/tbl2inc_c.pl \
 		src/libvterm/vterm.pc.in \
 		src/libvterm/doc/URLs \
 		src/libvterm/doc/seqs.txt \
+		src/libvterm/find-wide-chars.pl \
 		src/libvterm/include/vterm.h \
 		src/libvterm/include/vterm_keycodes.h \
 		src/libvterm/src/encoding.c \
@@ -312,6 +314,7 @@ SRC_ALL =	\
 		src/libvterm/src/encoding/DECdrawing.tbl \
 		src/libvterm/src/encoding/uk.inc \
 		src/libvterm/src/encoding/uk.tbl \
+		src/libvterm/src/fullwidth.inc \
 		src/libvterm/src/keyboard.c \
 		src/libvterm/src/mouse.c \
 		src/libvterm/src/parser.c \
@@ -342,16 +345,17 @@ SRC_ALL =	\
 		src/libvterm/t/27state_reset.test \
 		src/libvterm/t/28state_dbl_wh.test \
 		src/libvterm/t/29state_fallback.test \
-		src/libvterm/t/30pen.test \
-		src/libvterm/t/40screen_ascii.test \
-		src/libvterm/t/41screen_unicode.test \
-		src/libvterm/t/42screen_damage.test \
-		src/libvterm/t/43screen_resize.test \
-		src/libvterm/t/44screen_pen.test \
-		src/libvterm/t/45screen_protect.test \
-		src/libvterm/t/46screen_extent.test \
-		src/libvterm/t/47screen_dbl_wh.test \
-		src/libvterm/t/48screen_termprops.test \
+		src/libvterm/t/30state_pen.test \
+		src/libvterm/t/31state_rep.test \
+		src/libvterm/t/60screen_ascii.test \
+		src/libvterm/t/61screen_unicode.test \
+		src/libvterm/t/62screen_damage.test \
+		src/libvterm/t/63screen_resize.test \
+		src/libvterm/t/64screen_pen.test \
+		src/libvterm/t/65screen_protect.test \
+		src/libvterm/t/66screen_extent.test \
+		src/libvterm/t/67screen_dbl_wh.test \
+		src/libvterm/t/68screen_termprops.test \
 		src/libvterm/t/90vttest_01-movement-1.test \
 		src/libvterm/t/90vttest_01-movement-2.test \
 		src/libvterm/t/90vttest_01-movement-3.test \

runtime/doc/if_lua.txt

@@ -199,6 +199,15 @@ Vim evaluation and command execution, and others.
 				returns it. Note that the buffer is not set as
 				current.
 
+	vim.call({name} [,{args}])
+				Proxy to call Vim function named {name} with 
+				arguments {args}.  Example: >
+					:lua print(vim.call('has', 'timers'))
+<
+	vim.fn			Proxy to call Vim functions. Proxy methods are
+				created on demand.  Example: >
+					:lua print(vim.fn.has('timers'))
+<
 
 ==============================================================================
 3. List userdata					*lua-list*

runtime/doc/popup.txt

@@ -150,7 +150,7 @@ different:						*E863*
 - When the job ends, the popup window closes.
 - The popup window can be closed with `popup_close()`, the terminal buffer
   then becomes hidden.
-- It is not possible to enter Terminal-Normal mode.
+- It is not possible to open a second popup window with a terminal. *E861*
 - The default Pmenu color is only used for the border and padding.  To change
   the color of the terminal itself set the Terminal highlight group before
   creating the terminal.  Setting 'wincolor' later can work but requires the

src/Make_cyg_ming.mak

@@ -1246,6 +1246,7 @@ $(OUTDIR)/pathdef.o:	$(PATHDEF_SRC) $(INCL)
 
 CCCTERM = $(CC) -c $(CFLAGS) -Ilibvterm/include -DINLINE="" \
 	  -DVSNPRINTF=vim_vsnprintf \
+	  -DSNPRINTF=vim_snprintf \
 	  -DIS_COMBINING_FUNCTION=utf_iscomposing_uint \
 	  -DWCWIDTH_FUNCTION=utf_uint2cells \
 	  -DGET_SPECIAL_PTY_TYPE_FUNCTION=get_special_pty_type

src/Make_mvc.mak

@@ -1812,6 +1812,7 @@ $(OUTDIR)/glbl_ime.obj:	$(OUTDIR) glbl_ime.cpp  dimm.h $(INCL)
 
 CCCTERM = $(CC) $(CFLAGS) -Ilibvterm/include -DINLINE="" \
 	-DVSNPRINTF=vim_vsnprintf \
+	-DSNPRINTF=vim_snprintf \
 	-DIS_COMBINING_FUNCTION=utf_iscomposing_uint \
 	-DWCWIDTH_FUNCTION=utf_uint2cells \
 	-DGET_SPECIAL_PTY_TYPE_FUNCTION=get_special_pty_type \

src/Makefile

@@ -3537,6 +3537,7 @@ Makefile:
 # prefix vterm_ to avoid name clashes.
 CCCTERM = $(CCC_NF) $(VTERM_CFLAGS) $(ALL_CFLAGS) -DINLINE="" \
 	  -DVSNPRINTF=vim_vsnprintf \
+	  -DSNPRINTF=vim_snprintf \
 	  -DIS_COMBINING_FUNCTION=utf_iscomposing_uint \
 	  -DWCWIDTH_FUNCTION=utf_uint2cells
 
@@ -3561,7 +3562,7 @@ objects/vterm_screen.o: libvterm/src/screen.c $(TERM_DEPS)
 objects/vterm_state.o: libvterm/src/state.c $(TERM_DEPS)
 	$(CCCTERM) -o $@ libvterm/src/state.c
 
-objects/vterm_unicode.o: libvterm/src/unicode.c $(TERM_DEPS)
+objects/vterm_unicode.o: libvterm/src/unicode.c $(TERM_DEPS) libvterm/src/fullwidth.inc
 	$(CCCTERM) -o $@ libvterm/src/unicode.c
 
 objects/vterm_vterm.o: libvterm/src/vterm.c $(TERM_DEPS)

src/ex_docmd.c

@@ -2918,8 +2918,12 @@ parse_cmd_address(exarg_T *eap, char **errormsg, int silent)
 	{
 	    case ADDR_LINES:
 	    case ADDR_OTHER:
-		// default is current line number
-		eap->line2 = curwin->w_cursor.lnum;
+		// Default is the cursor line number.  Avoid using an invalid
+		// line number though.
+		if (curwin->w_cursor.lnum > curbuf->b_ml.ml_line_count)
+		    eap->line2 = curbuf->b_ml.ml_line_count;
+		else
+		    eap->line2 = curwin->w_cursor.lnum;
 		break;
 	    case ADDR_WINDOWS:
 		eap->line2 = CURRENT_WIN_NR;

src/globals.h

@@ -1171,7 +1171,7 @@ EXTERN volatile sig_atomic_t got_int INIT(= FALSE); // set to TRUE when interrup
 EXTERN int	term_console INIT(= FALSE); // set to TRUE when console used
 #endif
 EXTERN int	termcap_active INIT(= FALSE);	// set by starttermcap()
-EXTERN int	cur_tmode INIT(= TMODE_COOK);	// input terminal mode
+EXTERN tmode_T	cur_tmode INIT(= TMODE_COOK);	// input terminal mode
 EXTERN int	bangredo INIT(= FALSE);	    // set to TRUE with ! command
 EXTERN int	searchcmdlen;		    // length of previous search cmd
 #ifdef FEAT_SYN_HL

src/if_lua.c

@@ -119,6 +119,7 @@ static luaV_Funcref *luaV_pushfuncref(lua_State *L, char_u *name);
 #define luaL_buffinit dll_luaL_buffinit
 #define luaL_addlstring dll_luaL_addlstring
 #define luaL_pushresult dll_luaL_pushresult
+#define luaL_loadstring dll_luaL_loadstring
 // lua
 #if LUA_VERSION_NUM <= 501
 #define lua_tonumber dll_lua_tonumber
@@ -213,6 +214,7 @@ lua_State *(*dll_luaL_newstate) (void);
 void (*dll_luaL_buffinit) (lua_State *L, luaL_Buffer *B);
 void (*dll_luaL_addlstring) (luaL_Buffer *B, const char *s, size_t l);
 void (*dll_luaL_pushresult) (luaL_Buffer *B);
+int (*dll_luaL_loadstring) (lua_State *L, const char *s);
 // lua
 #if LUA_VERSION_NUM <= 501
 lua_Number (*dll_lua_tonumber) (lua_State *L, int idx);
@@ -325,6 +327,7 @@ static const luaV_Reg luaV_dll[] = {
     {"luaL_buffinit", (luaV_function) &dll_luaL_buffinit},
     {"luaL_addlstring", (luaV_function) &dll_luaL_addlstring},
     {"luaL_pushresult", (luaV_function) &dll_luaL_pushresult},
+    {"luaL_loadstring", (luaV_function) &dll_luaL_loadstring},
     // lua
 #if LUA_VERSION_NUM <= 501
     {"lua_tonumber", (luaV_function) &dll_lua_tonumber},
@@ -568,8 +571,21 @@ luaV_totypval(lua_State *L, int pos, typval_T *tv)
 	    break;
 	case LUA_TNUMBER:
 #ifdef FEAT_FLOAT
-	    tv->v_type = VAR_FLOAT;
-	    tv->vval.v_float = (float_T) lua_tonumber(L, pos);
+	{
+	    const lua_Number n = lua_tonumber(L, pos);
+
+	    if (n > (lua_Number)INT64_MAX || n < (lua_Number)INT64_MIN
+		    || ((lua_Number)((varnumber_T)n)) != n)
+	    {
+		tv->v_type = VAR_FLOAT;
+		tv->vval.v_float = (float_T)n;
+	    }
+	    else
+	    {
+		tv->v_type = VAR_NUMBER;
+		tv->vval.v_number = (varnumber_T)n;
+	    }
+	}
 #else
 	    tv->v_type = VAR_NUMBER;
 	    tv->vval.v_number = (varnumber_T) lua_tointeger(L, pos);
@@ -1903,6 +1919,52 @@ luaV_type(lua_State *L)
     return 1;
 }
 
+    static int
+luaV_call(lua_State *L)
+{
+    int		argc = lua_gettop(L) - 1;
+    size_t	funcname_len;
+    char_u	*funcname;
+    char	*error = NULL;
+    typval_T	rettv;
+    typval_T	argv[MAX_FUNC_ARGS + 1];
+    int		i = 0;
+
+    if (argc > MAX_FUNC_ARGS)
+	return luaL_error(L, "Function called with too many arguments");
+
+    funcname = (char_u *)luaL_checklstring(L, 1, &funcname_len);
+
+    for (; i < argc; i++)
+    {
+	if (luaV_totypval(L, i + 2, &argv[i]) == FAIL)
+	{
+	    error = "lua: cannot convert value";
+	    goto free_vim_args;
+	}
+    }
+
+    argv[argc].v_type = VAR_UNKNOWN;
+
+    if (call_vim_function(funcname, argc, argv, &rettv) == FAIL)
+    {
+	error = "lua: call_vim_function failed";
+	goto free_vim_args;
+    }
+
+    luaV_pushtypval(L, &rettv);
+    clear_tv(&rettv);
+
+free_vim_args:
+    while (i > 0)
+	clear_tv(&argv[--i]);
+
+    if (error == NULL)
+	return 1;
+    else
+	return luaL_error(L, error);
+}
+
 static const luaL_Reg luaV_module[] = {
     {"command", luaV_command},
     {"eval", luaV_eval},
@@ -1916,6 +1978,7 @@ static const luaL_Reg luaV_module[] = {
     {"window", luaV_window},
     {"open", luaV_open},
     {"type", luaV_type},
+    {"call", luaV_call},
     {NULL, NULL}
 };
 
@@ -1997,6 +2060,17 @@ luaV_setref(lua_State *L)
     return 1;
 }
 
+#define LUA_VIM_FN_CODE \
+    "vim.fn = setmetatable({}, {"\
+    "  __index = function (t, key)"\
+    "    local function _fn(...)"\
+    "      return vim.call(key, ...)"\
+    "    end"\
+    "    t[key] = _fn"\
+    "    return _fn"\
+    "  end"\
+    "})"
+
     static int
 luaopen_vim(lua_State *L)
 {
@@ -2052,6 +2126,8 @@ luaopen_vim(lua_State *L)
     lua_pushvalue(L, 1); // cache table
     luaV_openlib(L, luaV_module, 1);
     lua_setglobal(L, LUAVIM_NAME);
+    // custom code
+    (void)luaL_dostring(L, LUA_VIM_FN_CODE);
     return 0;
 }
 

src/json.c

@@ -20,6 +20,8 @@
 
 static int json_encode_item(garray_T *gap, typval_T *val, int copyID, int options);
 
+static char e_json_error[] = N_("E491: json decode error at '%s'");
+
 /*
  * Encode "val" into a JSON format string.
  * The result is added to "gap"
@@ -740,7 +742,7 @@ json_decode_item(js_read_T *reader, typval_T *res, int options)
 			retval = json_decode_string(reader, cur_item, *p);
 		    else
 		    {
-			emsg(_(e_invarg));
+			semsg(_(e_json_error), p);
 			retval = FAIL;
 		    }
 		    break;
@@ -748,7 +750,7 @@ json_decode_item(js_read_T *reader, typval_T *res, int options)
 		case ',': // comma: empty item
 		    if ((options & JSON_JS) == 0)
 		    {
-			emsg(_(e_invarg));
+			semsg(_(e_json_error), p);
 			retval = FAIL;
 			break;
 		    }
@@ -778,7 +780,7 @@ json_decode_item(js_read_T *reader, typval_T *res, int options)
 			    }
 			    if (!VIM_ISDIGIT(*sp))
 			    {
-				emsg(_(e_invarg));
+				semsg(_(e_json_error), p);
 				retval = FAIL;
 				break;
 			    }
@@ -809,7 +811,7 @@ json_decode_item(js_read_T *reader, typval_T *res, int options)
 				    &nr, NULL, 0, TRUE);
 			    if (len == 0)
 			    {
-				emsg(_(e_invarg));
+				semsg(_(e_json_error), p);
 				retval = FAIL;
 				goto theend;
 			    }
@@ -962,7 +964,7 @@ item_end:
 			retval = MAYBE;
 		    else
 		    {
-			emsg(_(e_invarg));
+			semsg(_(e_json_error), p);
 			retval = FAIL;
 		    }
 		    goto theend;
@@ -980,7 +982,7 @@ item_end:
 			retval = MAYBE;
 		    else
 		    {
-			emsg(_(e_invarg));
+			semsg(_(e_json_error), p);
 			retval = FAIL;
 		    }
 		    goto theend;
@@ -1036,7 +1038,7 @@ item_end:
 			retval = MAYBE;
 		    else
 		    {
-			emsg(_(e_invarg));
+			semsg(_(e_json_error), p);
 			retval = FAIL;
 		    }
 		    goto theend;
@@ -1055,7 +1057,7 @@ item_end:
 	res->v_type = VAR_SPECIAL;
 	res->vval.v_number = VVAL_NONE;
     }
-    emsg(_(e_invarg));
+    semsg(_(e_json_error), p);
 
 theend:
     ga_clear(&stack);
@@ -1079,7 +1081,7 @@ json_decode_all(js_read_T *reader, typval_T *res, int options)
     if (ret != OK)
     {
 	if (ret == MAYBE)
-	    emsg(_(e_invarg));
+	    semsg(_(e_json_error), reader->js_buf);
 	return FAIL;
     }
     json_skip_white(reader);

src/libvterm/CONTRIBUTING

@@ -0,0 +1,22 @@
+How to Contribute
+-----------------
+
+The main resources for this library are:
+
+  Launchpad
+    https://launchpad.net/libvterm
+
+  Freenode:
+    ##tty or #tickit on irc.freenode.net
+
+  Email:
+    Paul "LeoNerd" Evans <leonerd@leonerd.org.uk>
+
+
+Bug reports and feature requests can be sent to any of the above resources.
+
+New features, bug patches, etc.. should in the first instance be discussed via
+any of the resources listed above, before starting work on the actual code.
+There may be future plans or development already in-progress that could be
+affected so it is better to discuss the ideas first before starting work
+actually writing any code.

src/libvterm/Makefile

@@ -37,13 +37,13 @@ INCFILES=$(TBLFILES:.tbl=.inc)
 HFILES_INT=$(sort $(wildcard src/*.h)) $(HFILES)
 
 VERSION_MAJOR=0
-VERSION_MINOR=0
+VERSION_MINOR=1
 
 VERSION_CURRENT=0
 VERSION_REVISION=0
 VERSION_AGE=0
 
-VERSION=0
+VERSION=$(VERSION_MAJOR).$(VERSION_MINOR)
 
 PREFIX=/usr/local
 BINDIR=$(PREFIX)/bin
@@ -66,6 +66,9 @@ src/%.lo: src/%.c $(HFILES_INT)
 src/encoding/%.inc: src/encoding/%.tbl
 	perl -CSD tbl2inc_c.pl $< >$@
 
+src/fullwidth.inc:
+	@perl find-wide-chars.pl >$@
+
 src/encoding.lo: $(INCFILES)
 
 bin/%: bin/%.c $(LIBRARY)
@@ -107,15 +110,13 @@ install-bin: $(BINFILES)
 
 # DIST CUT
 
-VERSION=$(VERSION_MAJOR).$(VERSION_MINOR)
-
 DISTDIR=libvterm-$(VERSION)
 
 distdir: $(INCFILES)
 	mkdir __distdir
-	cp LICENSE __distdir
+	cp LICENSE CONTRIBUTING __distdir
 	mkdir __distdir/src
-	cp src/*.c src/*.h __distdir/src
+	cp src/*.c src/*.h src/*.inc __distdir/src
 	mkdir __distdir/src/encoding
 	cp src/encoding/*.inc __distdir/src/encoding
 	mkdir __distdir/include

src/libvterm/README

@@ -11,6 +11,9 @@ Modifications:
 - Convert from C99 to C90.
 - Other changes to support embedding in Vim.
 
+To find the diff of a libvterm patch edit this URL, changing "999" to the patch
+number:
+    https://bazaar.launchpad.net/~libvterm/libvterm/trunk/diff/999?context=3
 
 To merge in changes from Github, do this:
 - Commit any pending changes.

src/libvterm/doc/seqs.txt

@@ -97,6 +97,7 @@ between states.
       CSI Z            = CBT
       CSI `            = HPA
       CSI a            = HPR
+      CSI b            = REP
 123   CSI   c          = DA, device attributes
 123       0            =   DA
  23   CSI >   c        = DECSDA
@@ -166,8 +167,10 @@ between states.
 123   SGR 1            = Bold on
       SGR 3            = Italic on
 123   SGR 4            = Underline single
+      SGR 4:x          = Underline style
 123   SGR 5            = Blink on
 123   SGR 7            = Reverse on
+      SGR 8            = Conceal on
       SGR 9            = Strikethrough on
       SGR 10-19        = Select font
       SGR 21           = Underline double
@@ -176,6 +179,7 @@ between states.
  23   SGR 24           = Underline off
  23   SGR 25           = Blink off
  23   SGR 27           = Reverse off
+      SGR 28           = Conceal off
       SGR 29           = Strikethrough off
       SGR 30-37        = Foreground ANSI
       SGR 38           = Foreground alternative palette

src/libvterm/find-wide-chars.pl

@@ -0,0 +1,30 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+STDOUT->autoflush(1);
+
+sub iswide
+{
+   my ( $cp ) = @_;
+   return chr($cp) =~ m/\p{East_Asian_Width=Wide}|\p{East_Asian_Width=Fullwidth}/;
+}
+
+my ( $start, $end );
+foreach my $cp ( 0 .. 0x1FFFF ) {
+   iswide($cp) or next;
+
+   if( defined $end and $end == $cp-1 ) {
+      # extend the range
+      $end = $cp;
+      next;
+   }
+
+   # start a new range
+   printf "  { %#04x, %#04x },\n", $start, $end if defined $start;
+
+   $start = $end = $cp;
+}
+
+printf "  { %#04x, %#04x },\n", $start, $end if defined $start;

src/libvterm/include/vterm.h

@@ -19,6 +19,12 @@ extern "C" {
 typedef unsigned char		uint8_t;
 typedef unsigned int		uint32_t;
 
+#define VTERM_VERSION_MAJOR 0
+#define VTERM_VERSION_MINOR 1
+
+#define VTERM_CHECK_VERSION \
+        vterm_check_version(VTERM_VERSION_MAJOR, VTERM_VERSION_MINOR)
+
 typedef struct VTerm VTerm;
 typedef struct VTermState VTermState;
 typedef struct VTermScreen VTermScreen;
@@ -79,18 +85,107 @@ INLINE void vterm_rect_move(VTermRect *rect, int row_delta, int col_delta)
 }
 #endif
 
-// The ansi_index is used for the lower 16 colors, which can be set to any
-// color.
-#define VTERM_ANSI_INDEX_DEFAULT 0	// color cleared
-#define VTERM_ANSI_INDEX_MIN 1
-#define VTERM_ANSI_INDEX_MAX 16
-#define VTERM_ANSI_INDEX_NONE 255	// non-ANSI color, use red/green/blue
+/**
+ * Bit-field describing the value of VTermColor.type
+ */
+typedef enum {
+  /**
+   * If the lower bit of `type` is not set, the colour is 24-bit RGB.
+   */
+  VTERM_COLOR_RGB = 0x00,
+
+  /**
+   * The colour is an index into a palette of 256 colours.
+   */
+  VTERM_COLOR_INDEXED = 0x01,
+
+  /**
+   * Mask that can be used to extract the RGB/Indexed bit.
+   */
+  VTERM_COLOR_TYPE_MASK = 0x01,
+
+  /**
+   * If set, indicates that this colour should be the default foreground
+   * color, i.e. there was no SGR request for another colour. When
+   * rendering this colour it is possible to ignore "idx" and just use a
+   * colour that is not in the palette.
+   */
+  VTERM_COLOR_DEFAULT_FG = 0x02,
+
+  /**
+   * If set, indicates that this colour should be the default background
+   * color, i.e. there was no SGR request for another colour. A common
+   * option when rendering this colour is to not render a background at
+   * all, for example by rendering the window transparently at this spot.
+   */
+  VTERM_COLOR_DEFAULT_BG = 0x04,
+
+  /**
+   * Mask that can be used to extract the default foreground/background bit.
+   */
+  VTERM_COLOR_DEFAULT_MASK = 0x06
+} VTermColorType;
+
+/**
+ * Returns true if the VTERM_COLOR_RGB `type` flag is set, indicating that the
+ * given VTermColor instance is an indexed colour.
+ */
+#define VTERM_COLOR_IS_INDEXED(col) \
+  (((col)->type & VTERM_COLOR_TYPE_MASK) == VTERM_COLOR_INDEXED)
+
+/**
+ * Returns true if the VTERM_COLOR_INDEXED `type` flag is set, indicating that
+ * the given VTermColor instance is an rgb colour.
+ */
+#define VTERM_COLOR_IS_RGB(col) \
+  (((col)->type & VTERM_COLOR_TYPE_MASK) == VTERM_COLOR_RGB)
+
+/**
+ * Returns true if the VTERM_COLOR_DEFAULT_FG `type` flag is set, indicating
+ * that the given VTermColor instance corresponds to the default foreground
+ * color.
+ */
+#define VTERM_COLOR_IS_DEFAULT_FG(col) \
+  (!!((col)->type & VTERM_COLOR_DEFAULT_FG))
+
+/**
+ * Returns true if the VTERM_COLOR_DEFAULT_BG `type` flag is set, indicating
+ * that the given VTermColor instance corresponds to the default background
+ * color.
+ */
+#define VTERM_COLOR_IS_DEFAULT_BG(col) \
+  (!!((col)->type & VTERM_COLOR_DEFAULT_BG))
 
 typedef struct {
+  /**
+   * Tag indicating which member is actually valid.
+   * Please use the `VTERM_COLOR_IS_*` test macros to check whether a
+   * particular type flag is set.
+   */
+  uint8_t type;
+
   uint8_t red, green, blue;
-  uint8_t ansi_index;
+
+  uint8_t index;
 } VTermColor;
 
+/**
+ * Constructs a new VTermColor instance representing the given RGB values.
+ */
+void vterm_color_rgb(VTermColor *col, uint8_t red, uint8_t green, uint8_t blue);
+
+/**
+ * Construct a new VTermColor instance representing an indexed color with the
+ * given index.
+ */
+void vterm_color_indexed(VTermColor *col, uint8_t idx);
+
+/**
+ * Compares two colours. Returns true if the colors are equal, false otherwise.
+ */
+int vterm_color_is_equal(const VTermColor *a, const VTermColor *b);
+
+
 typedef enum {
   // VTERM_VALUETYPE_NONE = 0
   VTERM_VALUETYPE_BOOL = 1,
@@ -101,10 +196,17 @@ typedef enum {
   VTERM_N_VALUETYPES
 } VTermValueType;
 
+typedef struct {
+  const char *str;
+  size_t      len : 30;
+  unsigned int  initial : 1;
+  unsigned int  final : 1;
+} VTermStringFragment;
+
 typedef union {
   int boolean;
   int number;
-  char *string;
+  VTermStringFragment string;
   VTermColor color;
 } VTermValue;
 
@@ -115,6 +217,7 @@ typedef enum {
   VTERM_ATTR_ITALIC,     // bool:   3, 23
   VTERM_ATTR_BLINK,      // bool:   5, 25
   VTERM_ATTR_REVERSE,    // bool:   7, 27
+  VTERM_ATTR_CONCEAL,    // bool:   8, 28
   VTERM_ATTR_STRIKE,     // bool:   9, 29
   VTERM_ATTR_FONT,       // number: 10-19
   VTERM_ATTR_FOREGROUND, // color:  30-39 90-97
@@ -164,8 +267,9 @@ typedef struct {
 } VTermGlyphInfo;
 
 typedef struct {
-  unsigned int    doublewidth:1;     // DECDWL or DECDHL line
-  unsigned int    doubleheight:2;    // DECDHL line (1=top 2=bottom)
+  unsigned int    doublewidth:1;     /* DECDWL or DECDHL line */
+  unsigned int    doubleheight:2;    /* DECDHL line (1=top 2=bottom) */
+  unsigned int    continuation:1;    /* Line is a flow continuation of the previous */
 } VTermLineInfo;
 
 typedef struct {
@@ -175,6 +279,8 @@ typedef struct {
   void  (*free)(void *ptr, void *allocdata);
 } VTermAllocatorFunctions;
 
+void vterm_check_version(int major, int minor);
+
 // Allocate and initialize a new terminal with default allocators.
 VTerm *vterm_new(int rows, int cols);
 
@@ -194,10 +300,17 @@ void vterm_set_utf8(VTerm *vt, int is_utf8);
 
 size_t vterm_input_write(VTerm *vt, const char *bytes, size_t len);
 
+/* Setting output callback will override the buffer logic */
+typedef void VTermOutputCallback(const char *s, size_t len, void *user);
+void vterm_output_set_callback(VTerm *vt, VTermOutputCallback *func, void *user);
+
+/* These buffer functions only work if output callback is NOT set
+ * These are deprecated and will be removed in a later version */
 size_t vterm_output_get_buffer_size(const VTerm *vt);
 size_t vterm_output_get_buffer_current(const VTerm *vt);
 size_t vterm_output_get_buffer_remaining(const VTerm *vt);
 
+/* This too */
 size_t vterm_output_read(VTerm *vt, char *buffer, size_t len);
 
 int vterm_is_modify_other_keys(VTerm *vt);
@@ -241,8 +354,8 @@ typedef struct {
   int (*control)(unsigned char control, void *user);
   int (*escape)(const char *bytes, size_t len, void *user);
   int (*csi)(const char *leader, const long args[], int argcount, const char *intermed, char command, void *user);
-  int (*osc)(const char *command, size_t cmdlen, void *user);
-  int (*dcs)(const char *command, size_t cmdlen, void *user);
+  int (*osc)(int command, VTermStringFragment frag, void *user);
+  int (*dcs)(const char *command, size_t commandlen, VTermStringFragment frag, void *user);
   int (*resize)(int rows, int cols, void *user);
 } VTermParserCallbacks;
 
@@ -253,6 +366,15 @@ void *vterm_parser_get_cbdata(VTerm *vt);
 // State layer
 // -----------
 
+/* Copies of VTermState fields that the 'resize' callback might have reason to
+ * edit. 'resize' callback gets total control of these fields and may
+ * free-and-reallocate them if required. They will be copied back from the
+ * struct after the callback has returned.
+ */
+typedef struct {
+  VTermPos pos;                /* current cursor position */
+} VTermStateFields;
+
 typedef struct {
   int (*putglyph)(VTermGlyphInfo *info, VTermPos pos, void *user);
   int (*movecursor)(VTermPos pos, VTermPos oldpos, int visible, void *user);
@@ -265,7 +387,7 @@ typedef struct {
   // was accepted, 0 otherwise.
   int (*settermprop)(VTermProp prop, VTermValue *val, void *user);
   int (*bell)(void *user);
-  int (*resize)(int rows, int cols, VTermPos *delta, void *user);
+  int (*resize)(int rows, int cols, VTermStateFields *fields, void *user);
   int (*setlineinfo)(int row, const VTermLineInfo *newinfo, const VTermLineInfo *oldinfo, void *user);
 } VTermStateCallbacks;
 
@@ -282,13 +404,19 @@ typedef struct {
   // useful to add protocol?
 } VTermMouseState;
 
+typedef struct {
+  int (*control)(unsigned char control, void *user);
+  int (*csi)(const char *leader, const long args[], int argcount, const char *intermed, char command, void *user);
+  int (*osc)(int command, VTermStringFragment frag, void *user);
+  int (*dcs)(const char *command, size_t commandlen, VTermStringFragment frag, void *user);
+} VTermStateFallbacks;
+
 VTermState *vterm_obtain_state(VTerm *vt);
 
 void  vterm_state_set_callbacks(VTermState *state, const VTermStateCallbacks *callbacks, void *user);
 void *vterm_state_get_cbdata(VTermState *state);
 
-// Only invokes control, csi, osc, dcs
-void  vterm_state_set_unrecognised_fallbacks(VTermState *state, const VTermParserCallbacks *fallbacks, void *user);
+void  vterm_state_set_unrecognised_fallbacks(VTermState *state, const VTermStateFallbacks *fallbacks, void *user);
 void *vterm_state_get_unrecognised_fbdata(VTermState *state);
 
 // Initialize the state.
@@ -307,6 +435,18 @@ void vterm_state_focus_in(VTermState *state);
 void vterm_state_focus_out(VTermState *state);
 const VTermLineInfo *vterm_state_get_lineinfo(const VTermState *state, int row);
 
+/**
+ * Makes sure that the given color `col` is indeed an RGB colour. After this
+ * function returns, VTERM_COLOR_IS_RGB(col) will return true, while all other
+ * flags stored in `col->type` will have been reset.
+ *
+ * @param state is the VTermState instance from which the colour palette should
+ * be extracted.
+ * @param col is a pointer at the VTermColor instance that should be converted
+ * to an RGB colour.
+ */
+void vterm_state_convert_color_to_rgb(const VTermState *state, VTermColor *col);
+
 // ------------
 // Screen layer
 // ------------
@@ -317,12 +457,20 @@ typedef struct {
     unsigned int italic    : 1;
     unsigned int blink     : 1;
     unsigned int reverse   : 1;
+    unsigned int conceal   : 1;
     unsigned int strike    : 1;
     unsigned int font      : 4; // 0 to 9
     unsigned int dwl       : 1; // On a DECDWL or DECDHL line
     unsigned int dhl       : 2; // On a DECDHL line (1=top 2=bottom)
 } VTermScreenCellAttrs;
 
+enum {
+  VTERM_UNDERLINE_OFF,
+  VTERM_UNDERLINE_SINGLE,
+  VTERM_UNDERLINE_DOUBLE,
+  VTERM_UNDERLINE_CURLY,
+};
+
 typedef struct {
 #define VTERM_MAX_CHARS_PER_CELL 6
   uint32_t chars[VTERM_MAX_CHARS_PER_CELL];
@@ -356,8 +504,7 @@ VTermScreen *vterm_obtain_screen(VTerm *vt);
 void  vterm_screen_set_callbacks(VTermScreen *screen, const VTermScreenCallbacks *callbacks, void *user);
 void *vterm_screen_get_cbdata(VTermScreen *screen);
 
-// Only invokes control, csi, osc, dcs
-void  vterm_screen_set_unrecognised_fallbacks(VTermScreen *screen, const VTermParserCallbacks *fallbacks, void *user);
+void  vterm_screen_set_unrecognised_fallbacks(VTermScreen *screen, const VTermStateFallbacks *fallbacks, void *user);
 void *vterm_screen_get_unrecognised_fbdata(VTermScreen *screen);
 
 // Enable support for using the alternate screen if "altscreen" is non-zero.
@@ -399,8 +546,9 @@ typedef enum {
   VTERM_ATTR_FONT_MASK       = 1 << 6,
   VTERM_ATTR_FOREGROUND_MASK = 1 << 7,
   VTERM_ATTR_BACKGROUND_MASK = 1 << 8,
+  VTERM_ATTR_CONCEAL_MASK    = 1 << 9,
 
-  VTERM_ALL_ATTRS_MASK = (1 << 9) - 1
+  VTERM_ALL_ATTRS_MASK = (1 << 10) - 1
 } VTermAttrMask;
 
 int vterm_screen_get_attrs_extent(const VTermScreen *screen, VTermRect *extent, VTermPos pos, VTermAttrMask attrs);
@@ -409,6 +557,12 @@ int vterm_screen_get_cell(const VTermScreen *screen, VTermPos pos, VTermScreenCe
 
 int vterm_screen_is_eol(const VTermScreen *screen, VTermPos pos);
 
+/**
+ * Same as vterm_state_convert_color_to_rgb(), but takes a `screen` instead of a `state`
+ * instance.
+ */
+void vterm_screen_convert_color_to_rgb(const VTermScreen *screen, VTermColor *col);
+
 // ---------
 // Utilities
 // ---------

src/libvterm/src/fullwidth.inc

@@ -0,0 +1,111 @@
+  { 0x1100, 0x115f },
+  { 0x231a, 0x231b },
+  { 0x2329, 0x232a },
+  { 0x23e9, 0x23ec },
+  { 0x23f0, 0x23f0 },
+  { 0x23f3, 0x23f3 },
+  { 0x25fd, 0x25fe },
+  { 0x2614, 0x2615 },
+  { 0x2648, 0x2653 },
+  { 0x267f, 0x267f },
+  { 0x2693, 0x2693 },
+  { 0x26a1, 0x26a1 },
+  { 0x26aa, 0x26ab },
+  { 0x26bd, 0x26be },
+  { 0x26c4, 0x26c5 },
+  { 0x26ce, 0x26ce },
+  { 0x26d4, 0x26d4 },
+  { 0x26ea, 0x26ea },
+  { 0x26f2, 0x26f3 },
+  { 0x26f5, 0x26f5 },
+  { 0x26fa, 0x26fa },
+  { 0x26fd, 0x26fd },
+  { 0x2705, 0x2705 },
+  { 0x270a, 0x270b },
+  { 0x2728, 0x2728 },
+  { 0x274c, 0x274c },
+  { 0x274e, 0x274e },
+  { 0x2753, 0x2755 },
+  { 0x2757, 0x2757 },
+  { 0x2795, 0x2797 },
+  { 0x27b0, 0x27b0 },
+  { 0x27bf, 0x27bf },
+  { 0x2b1b, 0x2b1c },
+  { 0x2b50, 0x2b50 },
+  { 0x2b55, 0x2b55 },
+  { 0x2e80, 0x2e99 },
+  { 0x2e9b, 0x2ef3 },
+  { 0x2f00, 0x2fd5 },
+  { 0x2ff0, 0x2ffb },
+  { 0x3000, 0x303e },
+  { 0x3041, 0x3096 },
+  { 0x3099, 0x30ff },
+  { 0x3105, 0x312f },
+  { 0x3131, 0x318e },
+  { 0x3190, 0x31ba },
+  { 0x31c0, 0x31e3 },
+  { 0x31f0, 0x321e },
+  { 0x3220, 0x3247 },
+  { 0x3250, 0x4dbf },
+  { 0x4e00, 0xa48c },
+  { 0xa490, 0xa4c6 },
+  { 0xa960, 0xa97c },
+  { 0xac00, 0xd7a3 },
+  { 0xf900, 0xfaff },
+  { 0xfe10, 0xfe19 },
+  { 0xfe30, 0xfe52 },
+  { 0xfe54, 0xfe66 },
+  { 0xfe68, 0xfe6b },
+  { 0xff01, 0xff60 },
+  { 0xffe0, 0xffe6 },
+  { 0x16fe0, 0x16fe3 },
+  { 0x17000, 0x187f7 },
+  { 0x18800, 0x18af2 },
+  { 0x1b000, 0x1b11e },
+  { 0x1b150, 0x1b152 },
+  { 0x1b164, 0x1b167 },
+  { 0x1b170, 0x1b2fb },
+  { 0x1f004, 0x1f004 },
+  { 0x1f0cf, 0x1f0cf },
+  { 0x1f18e, 0x1f18e },
+  { 0x1f191, 0x1f19a },
+  { 0x1f200, 0x1f202 },
+  { 0x1f210, 0x1f23b },
+  { 0x1f240, 0x1f248 },
+  { 0x1f250, 0x1f251 },
+  { 0x1f260, 0x1f265 },
+  { 0x1f300, 0x1f320 },
+  { 0x1f32d, 0x1f335 },
+  { 0x1f337, 0x1f37c },
+  { 0x1f37e, 0x1f393 },
+  { 0x1f3a0, 0x1f3ca },
+  { 0x1f3cf, 0x1f3d3 },
+  { 0x1f3e0, 0x1f3f0 },
+  { 0x1f3f4, 0x1f3f4 },
+  { 0x1f3f8, 0x1f43e },
+  { 0x1f440, 0x1f440 },
+  { 0x1f442, 0x1f4fc },
+  { 0x1f4ff, 0x1f53d },
+  { 0x1f54b, 0x1f54e },
+  { 0x1f550, 0x1f567 },
+  { 0x1f57a, 0x1f57a },
+  { 0x1f595, 0x1f596 },
+  { 0x1f5a4, 0x1f5a4 },
+  { 0x1f5fb, 0x1f64f },
+  { 0x1f680, 0x1f6c5 },
+  { 0x1f6cc, 0x1f6cc },
+  { 0x1f6d0, 0x1f6d2 },
+  { 0x1f6d5, 0x1f6d5 },
+  { 0x1f6eb, 0x1f6ec },
+  { 0x1f6f4, 0x1f6fa },
+  { 0x1f7e0, 0x1f7eb },
+  { 0x1f90d, 0x1f971 },
+  { 0x1f973, 0x1f976 },
+  { 0x1f97a, 0x1f9a2 },
+  { 0x1f9a5, 0x1f9aa },
+  { 0x1f9ae, 0x1f9ca },
+  { 0x1f9cd, 0x1f9ff },
+  { 0x1fa70, 0x1fa73 },
+  { 0x1fa78, 0x1fa7a },
+  { 0x1fa80, 0x1fa82 },
+  { 0x1fa90, 0x1fa95 },

src/libvterm/src/keyboard.c

@@ -102,10 +102,10 @@ static keycodes_s keycodes[] = {
 
 static keycodes_s keycodes_fn[] = {
   { KEYCODE_NONE,       0, 0 },   // F0 - shouldn't happen
-  { KEYCODE_CSI_CURSOR, 'P', 0 }, // F1
-  { KEYCODE_CSI_CURSOR, 'Q', 0 }, // F2
-  { KEYCODE_CSI_CURSOR, 'R', 0 }, // F3
-  { KEYCODE_CSI_CURSOR, 'S', 0 }, // F4
+  { KEYCODE_SS3,	'P', 0 }, // F1
+  { KEYCODE_SS3,	'Q', 0 }, // F2
+  { KEYCODE_SS3,	'R', 0 }, // F3
+  { KEYCODE_SS3,	'S', 0 }, // F4
   { KEYCODE_CSINUM,     '~', 15 }, // F5
   { KEYCODE_CSINUM,     '~', 17 }, // F6
   { KEYCODE_CSINUM,     '~', 18 }, // F7

src/libvterm/src/parser.c

@@ -23,10 +23,10 @@ static void do_csi(VTerm *vt, char command)
 {
 #ifdef DEBUG_PARSER
   printf("Parsed CSI args as:\n", arglen, args);
-  printf(" leader: %s\n", vt->parser.csi_leader);
-  for(int argi = 0; argi < vt->parser.csi_argi; argi++) {
-    printf(" %lu", CSI_ARG(vt->parser.csi_args[argi]));
-    if(!CSI_ARG_HAS_MORE(vt->parser.csi_args[argi]))
+  printf(" leader: %s\n", vt->parser.v.csi.leader);
+  for(int argi = 0; argi < vt->parser.v.csi.argi; argi++) {
+    printf(" %lu", CSI_ARG(vt->parser.v.csi.args[argi]));
+    if(!CSI_ARG_HAS_MORE(vt->parser.v.csi.args[argi]))
       printf("\n");
   printf(" intermed: %s\n", vt->parser.intermed);
   }
@@ -34,9 +34,9 @@ static void do_csi(VTerm *vt, char command)
 
   if(vt->parser.callbacks && vt->parser.callbacks->csi)
     if((*vt->parser.callbacks->csi)(
-          vt->parser.csi_leaderlen ? vt->parser.csi_leader : NULL,
-          vt->parser.csi_args,
-          vt->parser.csi_argi,
+          vt->parser.v.csi.leaderlen ? vt->parser.v.csi.leader : NULL,
+          vt->parser.v.csi.args,
+          vt->parser.v.csi.argi,
           vt->parser.intermedlen ? vt->parser.intermed : NULL,
           command,
           vt->parser.cbdata))
@@ -61,65 +61,36 @@ static void do_escape(VTerm *vt, char command)
   DEBUG_LOG1("libvterm: Unhandled escape ESC 0x%02x\n", command);
 }
 
-static void append_strbuffer(VTerm *vt, const char *str, size_t len)
+static void string_fragment(VTerm *vt, const char *str, size_t len, int final)
 {
-  if(len > vt->parser.strbuffer_len - vt->parser.strbuffer_cur) {
-    len = vt->parser.strbuffer_len - vt->parser.strbuffer_cur;
-    DEBUG_LOG1("Truncating strbuffer preserve to %zu bytes\n", len);
+  VTermStringFragment frag;
+
+  frag.str = str;
+  frag.len = len;
+  frag.initial = vt->parser.string_initial;
+  frag.final = final;
+
+  switch(vt->parser.state) {
+    case OSC:
+      if(vt->parser.callbacks && vt->parser.callbacks->osc)
+        (*vt->parser.callbacks->osc)(vt->parser.v.osc.command, frag, vt->parser.cbdata);
+      break;
+
+    case DCS:
+      if(len && vt->parser.callbacks && vt->parser.callbacks->dcs)
+        (*vt->parser.callbacks->dcs)(vt->parser.v.dcs.command, vt->parser.v.dcs.commandlen, frag, vt->parser.cbdata);
+      break;
+
+    case NORMAL:
+    case CSI_LEADER:
+    case CSI_ARGS:
+    case CSI_INTERMED:
+    case OSC_COMMAND:
+    case DCS_COMMAND:
+      break;
   }
 
-  if(len > 0) {
-    strncpy(vt->parser.strbuffer + vt->parser.strbuffer_cur, str, len);
-    vt->parser.strbuffer_cur += len;
-  }
-}
-
-static void start_string(VTerm *vt, VTermParserStringType type)
-{
-  vt->parser.stringtype = type;
-
-  vt->parser.strbuffer_cur = 0;
-}
-
-static void more_string(VTerm *vt, const char *str, size_t len)
-{
-  append_strbuffer(vt, str, len);
-}
-
-static void done_string(VTerm *vt, const char *str, size_t len)
-{
-  if(vt->parser.strbuffer_cur) {
-    if(str)
-      append_strbuffer(vt, str, len);
-
-    str = vt->parser.strbuffer;
-    len = vt->parser.strbuffer_cur;
-  }
-  else if(!str) {
-    DEBUG_LOG("parser.c: TODO: No strbuffer _and_ no final fragment???\n");
-    len = 0;
-  }
-
-  switch(vt->parser.stringtype) {
-  case VTERM_PARSER_OSC:
-    if(vt->parser.callbacks && vt->parser.callbacks->osc)
-      if((*vt->parser.callbacks->osc)(str, len, vt->parser.cbdata))
-        return;
-
-    DEBUG_LOG2("libvterm: Unhandled OSC %.*s\n", (int)len, str);
-    return;
-
-  case VTERM_PARSER_DCS:
-    if(vt->parser.callbacks && vt->parser.callbacks->dcs)
-      if((*vt->parser.callbacks->dcs)(str, len, vt->parser.cbdata))
-        return;
-
-    DEBUG_LOG2("libvterm: Unhandled DCS %.*s\n", (int)len, str);
-    return;
-
-  case VTERM_N_PARSER_TYPES:
-    return;
-  }
+  vt->parser.string_initial = FALSE;
 }
 
 size_t vterm_input_write(VTerm *vt, const char *bytes, size_t len)
@@ -135,43 +106,47 @@ size_t vterm_input_write(VTerm *vt, const char *bytes, size_t len)
   case CSI_LEADER:
   case CSI_ARGS:
   case CSI_INTERMED:
-  case ESC:
+  case OSC_COMMAND:
+  case DCS_COMMAND:
     string_start = NULL;
     break;
-  case STRING:
-  case ESC_IN_STRING:
+  case OSC:
+  case DCS:
     string_start = bytes;
     break;
   }
 
-#define ENTER_STRING_STATE()   do { vt->parser.state = STRING; string_start = bytes + pos + 1; } while(0)
 #define ENTER_STATE(st)        do { vt->parser.state = st; string_start = NULL; } while(0)
 #define ENTER_NORMAL_STATE()   ENTER_STATE(NORMAL)
 
+#define IS_STRING_STATE()      (vt->parser.state >= OSC_COMMAND)
+
   for( ; pos < len; pos++) {
     unsigned char c = bytes[pos];
+    int c1_allowed = !vt->mode.utf8;
+    size_t string_len;
 
     if(c == 0x00 || c == 0x7f) { // NUL, DEL
-      if(vt->parser.state >= STRING) {
-        more_string(vt, string_start, bytes + pos - string_start);
+      if(IS_STRING_STATE()) {
+        string_fragment(vt, string_start, bytes + pos - string_start, FALSE);
         string_start = bytes + pos + 1;
       }
       continue;
     }
     if(c == 0x18 || c == 0x1a) { // CAN, SUB
+      vt->parser.in_esc = FALSE;
       ENTER_NORMAL_STATE();
       continue;
     }
     else if(c == 0x1b) { // ESC
       vt->parser.intermedlen = 0;
-      if(vt->parser.state == STRING)
-        vt->parser.state = ESC_IN_STRING;
-      else
-        ENTER_STATE(ESC);
+      if(!IS_STRING_STATE())
+        vt->parser.state = NORMAL;
+      vt->parser.in_esc = TRUE;
       continue;
     }
     else if(c == 0x07 &&  // BEL, can stand for ST in OSC or DCS state
-            vt->parser.state == STRING) {
+            IS_STRING_STATE()) {
       // fallthrough
     }
     else if(c < 0x20) { // other C0
@@ -182,96 +157,72 @@ size_t vterm_input_write(VTerm *vt, const char *bytes, size_t len)
           if(pos + 2 < len && bytes[pos + 1] == 0x20 && bytes[pos + 2] == 0x08)
             vt->in_backspace = 2; // Trigger when count down to 1
       }
-      if(vt->parser.state >= STRING)
-        more_string(vt, string_start, bytes + pos - string_start);
+      if(IS_STRING_STATE())
+        string_fragment(vt, string_start, bytes + pos - string_start, FALSE);
       do_control(vt, c);
-      if(vt->parser.state >= STRING)
+      if(IS_STRING_STATE())
         string_start = bytes + pos + 1;
       continue;
     }
     // else fallthrough
 
+    string_len = bytes + pos - string_start;
+
+    if(vt->parser.in_esc) {
+      // Hoist an ESC letter into a C1 if we're not in a string mode
+      // Always accept ESC \ == ST even in string mode
+      if(!vt->parser.intermedlen &&
+          c >= 0x40 && c < 0x60 &&
+          ((!IS_STRING_STATE() || c == 0x5c))) {
+        c += 0x40;
+        c1_allowed = TRUE;
+        string_len -= 1;
+        vt->parser.in_esc = FALSE;
+      }
+      else {
+        string_start = NULL;
+        vt->parser.state = NORMAL;
+      }
+    }
+
     switch(vt->parser.state) {
-    case ESC_IN_STRING:
-      if(c == 0x5c) { // ST
-        vt->parser.state = STRING;
-        done_string(vt, string_start, bytes + pos - string_start - 1);
-        ENTER_NORMAL_STATE();
-        break;
-      }
-      vt->parser.state = ESC;
-      // else fallthrough
-
-    case ESC:
-      switch(c) {
-      case 0x50: // DCS
-        start_string(vt, VTERM_PARSER_DCS);
-        ENTER_STRING_STATE();
-        break;
-      case 0x5b: // CSI
-        vt->parser.csi_leaderlen = 0;
-        ENTER_STATE(CSI_LEADER);
-        break;
-      case 0x5d: // OSC
-        start_string(vt, VTERM_PARSER_OSC);
-        ENTER_STRING_STATE();
-        break;
-      default:
-        if(is_intermed(c)) {
-          if(vt->parser.intermedlen < INTERMED_MAX-1)
-            vt->parser.intermed[vt->parser.intermedlen++] = c;
-        }
-        else if(!vt->parser.intermedlen && c >= 0x40 && c < 0x60) {
-          do_control(vt, c + 0x40);
-          ENTER_NORMAL_STATE();
-        }
-        else if(c >= 0x30 && c < 0x7f) {
-          do_escape(vt, c);
-          ENTER_NORMAL_STATE();
-        }
-        else {
-          DEBUG_LOG1("TODO: Unhandled byte %02x in Escape\n", c);
-        }
-      }
-      break;
-
     case CSI_LEADER:
       // Extract leader bytes 0x3c to 0x3f
       if(c >= 0x3c && c <= 0x3f) {
-        if(vt->parser.csi_leaderlen < CSI_LEADER_MAX-1)
-          vt->parser.csi_leader[vt->parser.csi_leaderlen++] = c;
+        if(vt->parser.v.csi.leaderlen < CSI_LEADER_MAX-1)
+          vt->parser.v.csi.leader[vt->parser.v.csi.leaderlen++] = c;
         break;
       }
 
       // else fallthrough
-      vt->parser.csi_leader[vt->parser.csi_leaderlen] = 0;
+      vt->parser.v.csi.leader[vt->parser.v.csi.leaderlen] = 0;
 
-      vt->parser.csi_argi = 0;
-      vt->parser.csi_args[0] = CSI_ARG_MISSING;
+      vt->parser.v.csi.argi = 0;
+      vt->parser.v.csi.args[0] = CSI_ARG_MISSING;
       vt->parser.state = CSI_ARGS;
 
       // fallthrough
     case CSI_ARGS:
       // Numerical value of argument
       if(c >= '0' && c <= '9') {
-        if(vt->parser.csi_args[vt->parser.csi_argi] == CSI_ARG_MISSING)
-          vt->parser.csi_args[vt->parser.csi_argi] = 0;
-        vt->parser.csi_args[vt->parser.csi_argi] *= 10;
-        vt->parser.csi_args[vt->parser.csi_argi] += c - '0';
+        if(vt->parser.v.csi.args[vt->parser.v.csi.argi] == CSI_ARG_MISSING)
+          vt->parser.v.csi.args[vt->parser.v.csi.argi] = 0;
+        vt->parser.v.csi.args[vt->parser.v.csi.argi] *= 10;
+        vt->parser.v.csi.args[vt->parser.v.csi.argi] += c - '0';
         break;
       }
       if(c == ':') {
-        vt->parser.csi_args[vt->parser.csi_argi] |= CSI_ARG_FLAG_MORE;
+        vt->parser.v.csi.args[vt->parser.v.csi.argi] |= CSI_ARG_FLAG_MORE;
         c = ';';
       }
       if(c == ';') {
-        vt->parser.csi_argi++;
-        vt->parser.csi_args[vt->parser.csi_argi] = CSI_ARG_MISSING;
+        vt->parser.v.csi.argi++;
+        vt->parser.v.csi.args[vt->parser.v.csi.argi] = CSI_ARG_MISSING;
         break;
       }
 
       // else fallthrough
-      vt->parser.csi_argi++;
+      vt->parser.v.csi.argi++;
       vt->parser.intermedlen = 0;
       vt->parser.state = CSI_INTERMED;
       // fallthrough
@@ -293,31 +244,79 @@ size_t vterm_input_write(VTerm *vt, const char *bytes, size_t len)
       ENTER_NORMAL_STATE();
       break;
 
-    case STRING:
-      if(c == 0x07 || (c == 0x9c && !vt->mode.utf8)) {
-        done_string(vt, string_start, bytes + pos - string_start);
-        ENTER_NORMAL_STATE();
+    case OSC_COMMAND:
+      /* Numerical value of command */
+      if(c >= '0' && c <= '9') {
+        if(vt->parser.v.osc.command == -1)
+          vt->parser.v.osc.command = 0;
+        else
+          vt->parser.v.osc.command *= 10;
+        vt->parser.v.osc.command += c - '0';
+        break;
       }
-      else if (pos + 1 == len) {
-	// end of input but OSC string isn't finished yet, copy it to
-	// vt->parser.strbuffer to continue it later
-        more_string(vt, string_start, bytes + pos + 1 - string_start);
+      if(c == ';') {
+        vt->parser.state = OSC;
+        string_start = bytes + pos + 1;
+        break;
+      }
+
+      /* else fallthrough */
+      string_start = bytes + pos;
+      string_len   = 0;
+      vt->parser.state = OSC;
+      goto string_state;
+
+    case DCS_COMMAND:
+      if(vt->parser.v.dcs.commandlen < CSI_LEADER_MAX)
+        vt->parser.v.dcs.command[vt->parser.v.dcs.commandlen++] = c;
+
+      if(c >= 0x40 && c<= 0x7e) {
+        string_start = bytes + pos + 1;
+        vt->parser.state = DCS;
+      }
+      break;
+
+string_state:
+    case OSC:
+    case DCS:
+      if(c == 0x07 || (c1_allowed && c == 0x9c)) {
+        string_fragment(vt, string_start, string_len, TRUE);
+        ENTER_NORMAL_STATE();
       }
       break;
 
     case NORMAL:
-      if(c >= 0x80 && c < 0xa0 && !vt->mode.utf8) {
+      if(vt->parser.in_esc) {
+        if(is_intermed(c)) {
+          if(vt->parser.intermedlen < INTERMED_MAX-1)
+            vt->parser.intermed[vt->parser.intermedlen++] = c;
+        }
+        else if(c >= 0x30 && c < 0x7f) {
+          do_escape(vt, c);
+          vt->parser.in_esc = 0;
+          ENTER_NORMAL_STATE();
+        }
+        else {
+          DEBUG_LOG1("TODO: Unhandled byte %02x in Escape\n", c);
+        }
+        break;
+      }
+      if(c1_allowed && c >= 0x80 && c < 0xa0) {
         switch(c) {
         case 0x90: // DCS
-          start_string(vt, VTERM_PARSER_DCS);
-          ENTER_STRING_STATE();
+          vt->parser.string_initial = TRUE;
+          vt->parser.v.dcs.commandlen = 0;
+          ENTER_STATE(DCS_COMMAND);
           break;
         case 0x9b: // CSI
+          vt->parser.v.csi.leaderlen = 0;
           ENTER_STATE(CSI_LEADER);
           break;
         case 0x9d: // OSC
-          start_string(vt, VTERM_PARSER_OSC);
-          ENTER_STRING_STATE();
+          vt->parser.v.osc.command = -1;
+          vt->parser.string_initial = TRUE;
+          string_start = bytes + pos + 1;
+          ENTER_STATE(OSC_COMMAND);
           break;
         default:
           do_control(vt, c);
@@ -341,6 +340,9 @@ size_t vterm_input_write(VTerm *vt, const char *bytes, size_t len)
     }
   }
 
+  if(string_start)
+    string_fragment(vt, string_start, bytes + pos - string_start, FALSE);
+
   return len;
 }
 

src/libvterm/src/pen.c

@@ -2,26 +2,34 @@
 
 #include <stdio.h>
 
-static const VTermColor ansi_colors[] = {
-  // R    G    B   index
-  {   0,   0,   0,  1 }, // black
-  { 224,   0,   0,  2 }, // red
-  {   0, 224,   0,  3 }, // green
-  { 224, 224,   0,  4 }, // yellow
-  {   0,   0, 224,  5 }, // blue
-  { 224,   0, 224,  6 }, // magenta
-  {   0, 224, 224,  7 }, // cyan
-  { 224, 224, 224,  8 }, // white == light grey
+/**
+ * Structure used to store RGB triples without the additional metadata stored in
+ * VTermColor.
+ */
+typedef struct {
+  uint8_t red, green, blue;
+} VTermRGB;
+
+static const VTermRGB ansi_colors[] = {
+  // R    G    B
+  {   0,   0,   0 }, // black
+  { 224,   0,   0 }, // red
+  {   0, 224,   0 }, // green
+  { 224, 224,   0 }, // yellow
+  {   0,   0, 224 }, // blue
+  { 224,   0, 224 }, // magenta
+  {   0, 224, 224 }, // cyan
+  { 224, 224, 224 }, // white == light grey
 
   // high intensity
-  { 128, 128, 128,  9 }, // black
-  { 255,  64,  64, 10 }, // red
-  {  64, 255,  64, 11 }, // green
-  { 255, 255,  64, 12 }, // yellow
-  {  64,  64, 255, 13 }, // blue
-  { 255,  64, 255, 14 }, // magenta
-  {  64, 255, 255, 15 }, // cyan
-  { 255, 255, 255, 16 }, // white for real
+  { 128, 128, 128 }, // black
+  { 255,  64,  64 }, // red
+  {  64, 255,  64 }, // green
+  { 255, 255,  64 }, // yellow
+  {  64,  64, 255 }, // blue
+  { 255,  64, 255 }, // magenta
+  {  64, 255, 255 }, // cyan
+  { 255, 255, 255 }, // white for real
 };
 
 static int ramp6[] = {
@@ -34,6 +42,15 @@ static int ramp24[] = {
   0x81, 0x8A, 0x94, 0x9E, 0xA8, 0xB2, 0xBC, 0xC6, 0xD0, 0xDA, 0xE4, 0xEE,
 };
 
+static void lookup_default_colour_ansi(long idx, VTermColor *col)
+{
+  vterm_color_rgb(
+      col,
+      ansi_colors[idx].red, ansi_colors[idx].green, ansi_colors[idx].blue);
+  col->index = idx;
+  col->type = VTERM_COLOR_INDEXED;
+}
+
 static int lookup_colour_ansi(const VTermState *state, long index, VTermColor *col)
 {
   if(index >= 0 && index < 16) {
@@ -54,10 +71,9 @@ static int lookup_colour_palette(const VTermState *state, long index, VTermColor
     // 216-colour cube
     index -= 16;
 
-    col->blue  = ramp6[index     % 6];
-    col->green = ramp6[index/6   % 6];
-    col->red   = ramp6[index/6/6 % 6];
-    col->ansi_index = VTERM_ANSI_INDEX_NONE;
+    vterm_color_rgb(col, ramp6[index/6/6 % 6],
+                         ramp6[index/6   % 6],
+                         ramp6[index     % 6]);
 
     return TRUE;
   }
@@ -65,10 +81,7 @@ static int lookup_colour_palette(const VTermState *state, long index, VTermColor
     // 24 greyscales
     index -= 232;
 
-    col->blue  = ramp24[index];
-    col->green = ramp24[index];
-    col->red   = ramp24[index];
-    col->ansi_index = VTERM_ANSI_INDEX_NONE;
+    vterm_color_rgb(col, ramp24[index], ramp24[index], ramp24[index]);
 
     return TRUE;
   }
@@ -76,25 +89,23 @@ static int lookup_colour_palette(const VTermState *state, long index, VTermColor
   return FALSE;
 }
 
-static int lookup_colour(const VTermState *state, int palette, const long args[], int argcount, VTermColor *col, int *index)
+static int lookup_colour(const VTermState *state, int palette, const long args[], int argcount, VTermColor *col)
 {
   switch(palette) {
   case 2: // RGB mode - 3 args contain colour values directly
     if(argcount < 3)
       return argcount;
 
-    col->red   = (uint8_t)CSI_ARG(args[0]);
-    col->green = (uint8_t)CSI_ARG(args[1]);
-    col->blue  = (uint8_t)CSI_ARG(args[2]);
-    col->ansi_index = VTERM_ANSI_INDEX_NONE;
+    vterm_color_rgb(col, CSI_ARG(args[0]), CSI_ARG(args[1]), CSI_ARG(args[2]));
 
     return 3;
 
   case 5: // XTerm 256-colour mode
-    if(index)
-      *index = CSI_ARG_OR(args[0], -1);
+    if (!argcount || CSI_ARG_IS_MISSING(args[0])) {
+      return argcount ? 1 : 0;
+    }
 
-    lookup_colour_palette(state, argcount ? CSI_ARG_OR(args[0], -1) : -1, col);
+    lookup_colour_palette(state, args[0], col);
 
     return argcount ? 1 : 0;
 
@@ -154,13 +165,12 @@ INTERNAL void vterm_state_newpen(VTermState *state)
   int col;
 
   // 90% grey so that pure white is brighter
-  state->default_fg.red = state->default_fg.green = state->default_fg.blue = 240;
-  state->default_fg.ansi_index = VTERM_ANSI_INDEX_DEFAULT;
-  state->default_bg.red = state->default_bg.green = state->default_bg.blue = 0;
-  state->default_bg.ansi_index = VTERM_ANSI_INDEX_DEFAULT;
+  vterm_color_rgb(&state->default_fg, 240, 240, 240);
+  vterm_color_rgb(&state->default_bg, 0, 0, 0);
+  vterm_state_set_default_colors(state, &state->default_fg, &state->default_bg);
 
   for(col = 0; col < 16; col++)
-    state->colors[col] = ansi_colors[col];
+    lookup_default_colour_ansi(col, &state->colors[col]);
 }
 
 INTERNAL void vterm_state_resetpen(VTermState *state)
@@ -170,11 +180,10 @@ INTERNAL void vterm_state_resetpen(VTermState *state)
   state->pen.italic = 0;    setpenattr_bool(state, VTERM_ATTR_ITALIC, 0);
   state->pen.blink = 0;     setpenattr_bool(state, VTERM_ATTR_BLINK, 0);
   state->pen.reverse = 0;   setpenattr_bool(state, VTERM_ATTR_REVERSE, 0);
+  state->pen.conceal = 0;   setpenattr_bool(state, VTERM_ATTR_CONCEAL, 0);
   state->pen.strike = 0;    setpenattr_bool(state, VTERM_ATTR_STRIKE, 0);
   state->pen.font = 0;      setpenattr_int( state, VTERM_ATTR_FONT, 0);
 
-  state->fg_index = -1;
-  state->bg_index = -1;
   state->pen.fg = state->default_fg;  setpenattr_col(state, VTERM_ATTR_FOREGROUND, state->default_fg);
   state->pen.bg = state->default_bg;  setpenattr_col(state, VTERM_ATTR_BACKGROUND, state->default_bg);
 }
@@ -192,6 +201,7 @@ INTERNAL void vterm_state_savepen(VTermState *state, int save)
     setpenattr_bool(state, VTERM_ATTR_ITALIC,     state->pen.italic);
     setpenattr_bool(state, VTERM_ATTR_BLINK,      state->pen.blink);
     setpenattr_bool(state, VTERM_ATTR_REVERSE,    state->pen.reverse);
+    setpenattr_bool(state, VTERM_ATTR_CONCEAL,    state->pen.conceal);
     setpenattr_bool(state, VTERM_ATTR_STRIKE,     state->pen.strike);
     setpenattr_int( state, VTERM_ATTR_FONT,       state->pen.font);
     setpenattr_col( state, VTERM_ATTR_FOREGROUND, state->pen.fg);
@@ -199,6 +209,40 @@ INTERNAL void vterm_state_savepen(VTermState *state, int save)
   }
 }
 
+void vterm_color_rgb(VTermColor *col, uint8_t red, uint8_t green, uint8_t blue)
+{
+  col->type = VTERM_COLOR_RGB;
+  col->red   = red;
+  col->green = green;
+  col->blue  = blue;
+}
+
+void vterm_color_indexed(VTermColor *col, uint8_t idx)
+{
+  col->type = VTERM_COLOR_INDEXED;
+  col->index = idx;
+}
+
+int vterm_color_is_equal(const VTermColor *a, const VTermColor *b)
+{
+  /* First make sure that the two colours are of the same type (RGB/Indexed) */
+  if (a->type != b->type) {
+    return FALSE;
+  }
+
+  /* Depending on the type inspect the corresponding members */
+  if (VTERM_COLOR_IS_INDEXED(a)) {
+    return a->index == b->index;
+  }
+  else if (VTERM_COLOR_IS_RGB(a)) {
+    return    (a->red   == b->red)
+           && (a->green == b->green)
+           && (a->blue  == b->blue);
+  }
+
+  return 0;
+}
+
 void vterm_state_get_default_colors(const VTermState *state, VTermColor *default_fg, VTermColor *default_bg)
 {
   *default_fg = state->default_fg;
@@ -212,8 +256,15 @@ void vterm_state_get_palette_color(const VTermState *state, int index, VTermColo
 
 void vterm_state_set_default_colors(VTermState *state, const VTermColor *default_fg, const VTermColor *default_bg)
 {
+  /* Copy the given colors */
   state->default_fg = *default_fg;
   state->default_bg = *default_bg;
+
+  /* Make sure the correct type flags are set */
+  state->default_fg.type = (state->default_fg.type & ~VTERM_COLOR_DEFAULT_MASK)
+                         | VTERM_COLOR_DEFAULT_FG;
+  state->default_bg.type = (state->default_bg.type & ~VTERM_COLOR_DEFAULT_MASK)
+                         | VTERM_COLOR_DEFAULT_BG;
 }
 
 void vterm_state_set_palette_color(VTermState *state, int index, const VTermColor *col)
@@ -221,10 +272,18 @@ void vterm_state_set_palette_color(VTermState *state, int index, const VTermColo
   if(index >= 0 && index < 16)
   {
     state->colors[index] = *col;
-    state->colors[index].ansi_index = index + VTERM_ANSI_INDEX_MIN;
+    state->colors[index].index = index + 1;
   }
 }
 
+void vterm_state_convert_color_to_rgb(const VTermState *state, VTermColor *col)
+{
+  if (VTERM_COLOR_IS_INDEXED(col)) { /* Convert indexed colors to RGB */
+    lookup_colour_palette(state, col->index, col);
+  }
+  col->type &= VTERM_COLOR_TYPE_MASK; /* Reset any metadata but the type */
+}
+
 void vterm_state_set_bold_highbright(VTermState *state, int bold_is_highbright)
 {
   state->bold_is_highbright = bold_is_highbright;
@@ -249,21 +308,40 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
       vterm_state_resetpen(state);
       break;
 
-    case 1: // Bold on
+    case 1: { // Bold on
+      const VTermColor *fg = &state->pen.fg;
       state->pen.bold = 1;
       setpenattr_bool(state, VTERM_ATTR_BOLD, 1);
-      if(state->fg_index > -1 && state->fg_index < 8 && state->bold_is_highbright)
-        set_pen_col_ansi(state, VTERM_ATTR_FOREGROUND, state->fg_index + (state->pen.bold ? 8 : 0));
+      if(!VTERM_COLOR_IS_DEFAULT_FG(fg) && VTERM_COLOR_IS_INDEXED(fg) && fg->index < 8 && state->bold_is_highbright)
+        set_pen_col_ansi(state, VTERM_ATTR_FOREGROUND, fg->index + (state->pen.bold ? 8 : 0));
       break;
+    }
 
     case 3: // Italic on
       state->pen.italic = 1;
       setpenattr_bool(state, VTERM_ATTR_ITALIC, 1);
       break;
 
-    case 4: // Underline single
-      state->pen.underline = 1;
-      setpenattr_int(state, VTERM_ATTR_UNDERLINE, 1);
+    case 4: // Underline
+      state->pen.underline = VTERM_UNDERLINE_SINGLE;
+      if(CSI_ARG_HAS_MORE(args[argi])) {
+        argi++;
+        switch(CSI_ARG(args[argi])) {
+          case 0:
+            state->pen.underline = 0;
+            break;
+          case 1:
+            state->pen.underline = VTERM_UNDERLINE_SINGLE;
+            break;
+          case 2:
+            state->pen.underline = VTERM_UNDERLINE_DOUBLE;
+            break;
+          case 3:
+            state->pen.underline = VTERM_UNDERLINE_CURLY;
+            break;
+        }
+      }
+      setpenattr_int(state, VTERM_ATTR_UNDERLINE, state->pen.underline);
       break;
 
     case 5: // Blink
@@ -276,6 +354,11 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
       setpenattr_bool(state, VTERM_ATTR_REVERSE, 1);
       break;
 
+    case 8: // Conceal on
+      state->pen.conceal = 1;
+      setpenattr_bool(state, VTERM_ATTR_CONCEAL, 1);
+      break;
+
     case 9: // Strikethrough on
       state->pen.strike = 1;
       setpenattr_bool(state, VTERM_ATTR_STRIKE, 1);
@@ -288,8 +371,8 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
       break;
 
     case 21: // Underline double
-      state->pen.underline = 2;
-      setpenattr_int(state, VTERM_ATTR_UNDERLINE, 2);
+      state->pen.underline = VTERM_UNDERLINE_DOUBLE;
+      setpenattr_int(state, VTERM_ATTR_UNDERLINE, state->pen.underline);
       break;
 
     case 22: // Bold off
@@ -317,6 +400,11 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
       setpenattr_bool(state, VTERM_ATTR_REVERSE, 0);
       break;
 
+    case 28: // Conceal off (Reveal)
+      state->pen.conceal = 0;
+      setpenattr_bool(state, VTERM_ATTR_CONCEAL, 0);
+      break;
+
     case 29: // Strikethrough off
       state->pen.strike = 0;
       setpenattr_bool(state, VTERM_ATTR_STRIKE, 0);
@@ -325,22 +413,19 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
     case 30: case 31: case 32: case 33:
     case 34: case 35: case 36: case 37: // Foreground colour palette
       value = CSI_ARG(args[argi]) - 30;
-      state->fg_index = value;
       if(state->pen.bold && state->bold_is_highbright)
         value += 8;
       set_pen_col_ansi(state, VTERM_ATTR_FOREGROUND, value);
       break;
 
     case 38: // Foreground colour alternative palette
-      state->fg_index = -1;
       if(argcount - argi < 1)
         return;
-      argi += 1 + lookup_colour(state, CSI_ARG(args[argi+1]), args+argi+2, argcount-argi-2, &state->pen.fg, &state->fg_index);
+      argi += 1 + lookup_colour(state, CSI_ARG(args[argi+1]), args+argi+2, argcount-argi-2, &state->pen.fg);
       setpenattr_col(state, VTERM_ATTR_FOREGROUND, state->pen.fg);
       break;
 
     case 39: // Foreground colour default
-      state->fg_index = -1;
       state->pen.fg = state->default_fg;
       setpenattr_col(state, VTERM_ATTR_FOREGROUND, state->pen.fg);
       break;
@@ -348,20 +433,17 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
     case 40: case 41: case 42: case 43:
     case 44: case 45: case 46: case 47: // Background colour palette
       value = CSI_ARG(args[argi]) - 40;
-      state->bg_index = value;
       set_pen_col_ansi(state, VTERM_ATTR_BACKGROUND, value);
       break;
 
     case 48: // Background colour alternative palette
-      state->bg_index = -1;
       if(argcount - argi < 1)
         return;
-      argi += 1 + lookup_colour(state, CSI_ARG(args[argi+1]), args+argi+2, argcount-argi-2, &state->pen.bg, &state->bg_index);
+      argi += 1 + lookup_colour(state, CSI_ARG(args[argi+1]), args+argi+2, argcount-argi-2, &state->pen.bg);
       setpenattr_col(state, VTERM_ATTR_BACKGROUND, state->pen.bg);
       break;
 
     case 49: // Default background
-      state->bg_index = -1;
       state->pen.bg = state->default_bg;
       setpenattr_col(state, VTERM_ATTR_BACKGROUND, state->pen.bg);
       break;
@@ -369,14 +451,12 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
     case 90: case 91: case 92: case 93:
     case 94: case 95: case 96: case 97: // Foreground colour high-intensity palette
       value = CSI_ARG(args[argi]) - 90 + 8;
-      state->fg_index = value;
       set_pen_col_ansi(state, VTERM_ATTR_FOREGROUND, value);
       break;
 
     case 100: case 101: case 102: case 103:
     case 104: case 105: case 106: case 107: // Background colour high-intensity palette
       value = CSI_ARG(args[argi]) - 100 + 8;
-      state->bg_index = value;
       set_pen_col_ansi(state, VTERM_ATTR_BACKGROUND, value);
       break;
 
@@ -395,6 +475,39 @@ INTERNAL void vterm_state_setpen(VTermState *state, const long args[], int argco
   }
 }
 
+static int vterm_state_getpen_color(const VTermColor *col, int argi, long args[], int fg)
+{
+    /* Do nothing if the given color is the default color */
+    if (( fg && VTERM_COLOR_IS_DEFAULT_FG(col)) ||
+        (!fg && VTERM_COLOR_IS_DEFAULT_BG(col))) {
+        return argi;
+    }
+
+    /* Decide whether to send an indexed color or an RGB color */
+    if (VTERM_COLOR_IS_INDEXED(col)) {
+        const uint8_t idx = col->index;
+        if (idx < 8) {
+            args[argi++] = (idx + (fg ? 30 : 40));
+        }
+        else if (idx < 16) {
+            args[argi++] = (idx - 8 + (fg ? 90 : 100));
+        }
+        else {
+            args[argi++] = CSI_ARG_FLAG_MORE | (fg ? 38 : 48);
+            args[argi++] = CSI_ARG_FLAG_MORE | 5;
+            args[argi++] = idx;
+        }
+    }
+    else if (VTERM_COLOR_IS_RGB(col)) {
+        args[argi++] = CSI_ARG_FLAG_MORE | (fg ? 38 : 48);
+        args[argi++] = CSI_ARG_FLAG_MORE | 2;
+        args[argi++] = CSI_ARG_FLAG_MORE | col->red;
+        args[argi++] = CSI_ARG_FLAG_MORE | col->green;
+        args[argi++] = col->blue;
+    }
+    return argi;
+}
+
 INTERNAL int vterm_state_getpen(VTermState *state, long args[], int argcount UNUSED)
 {
   int argi = 0;
@@ -405,8 +518,10 @@ INTERNAL int vterm_state_getpen(VTermState *state, long args[], int argcount UNU
   if(state->pen.italic)
     args[argi++] = 3;
 
-  if(state->pen.underline == 1)
+  if(state->pen.underline == VTERM_UNDERLINE_SINGLE)
     args[argi++] = 4;
+  if(state->pen.underline == VTERM_UNDERLINE_CURLY)
+    args[argi++] = 4 | CSI_ARG_FLAG_MORE, args[argi++] = 3;
 
   if(state->pen.blink)
     args[argi++] = 5;
@@ -414,58 +529,21 @@ INTERNAL int vterm_state_getpen(VTermState *state, long args[], int argcount UNU
   if(state->pen.reverse)
     args[argi++] = 7;
 
+  if(state->pen.conceal)
+    args[argi++] = 8;
+
   if(state->pen.strike)
     args[argi++] = 9;
 
   if(state->pen.font)
     args[argi++] = 10 + state->pen.font;
 
-  if(state->pen.underline == 2)
+  if(state->pen.underline == VTERM_UNDERLINE_DOUBLE)
     args[argi++] = 21;
 
-  if(state->fg_index >= 0 && state->fg_index < 8)
-    args[argi++] = 30 + state->fg_index;
-  else if(state->fg_index >= 8 && state->fg_index < 16)
-    args[argi++] = 90 + state->fg_index - 8;
-  else if(state->fg_index >= 16 && state->fg_index < 256) {
-    args[argi++] = CSI_ARG_FLAG_MORE|38;
-    args[argi++] = CSI_ARG_FLAG_MORE|5;
-    args[argi++] = state->fg_index;
-  }
-  else if(state->fg_index == -1) {
-    // Send palette 2 if the actual FG colour is not default
-    if(state->pen.fg.red   != state->default_fg.red   ||
-       state->pen.fg.green != state->default_fg.green ||
-       state->pen.fg.blue  != state->default_fg.blue  ) {
-      args[argi++] = CSI_ARG_FLAG_MORE|38;
-      args[argi++] = CSI_ARG_FLAG_MORE|2;
-      args[argi++] = CSI_ARG_FLAG_MORE | state->pen.fg.red;
-      args[argi++] = CSI_ARG_FLAG_MORE | state->pen.fg.green;
-      args[argi++] = state->pen.fg.blue;
-    }
-  }
+  argi = vterm_state_getpen_color(&state->pen.fg, argi, args, TRUE);
 
-  if(state->bg_index >= 0 && state->bg_index < 8)
-    args[argi++] = 40 + state->bg_index;
-  else if(state->bg_index >= 8 && state->bg_index < 16)
-    args[argi++] = 100 + state->bg_index - 8;
-  else if(state->bg_index >= 16 && state->bg_index < 256) {
-    args[argi++] = CSI_ARG_FLAG_MORE|48;
-    args[argi++] = CSI_ARG_FLAG_MORE|5;
-    args[argi++] = state->bg_index;
-  }
-  else if(state->bg_index == -1) {
-    // Send palette 2 if the actual BG colour is not default
-    if(state->pen.bg.red   != state->default_bg.red   ||
-       state->pen.bg.green != state->default_bg.green ||
-       state->pen.bg.blue  != state->default_bg.blue  ) {
-      args[argi++] = CSI_ARG_FLAG_MORE|48;
-      args[argi++] = CSI_ARG_FLAG_MORE|2;
-      args[argi++] = CSI_ARG_FLAG_MORE | state->pen.bg.red;
-      args[argi++] = CSI_ARG_FLAG_MORE | state->pen.bg.green;
-      args[argi++] = state->pen.bg.blue;
-    }
-  }
+  argi = vterm_state_getpen_color(&state->pen.bg, argi, args, FALSE);
 
   return argi;
 }
@@ -493,6 +571,10 @@ int vterm_state_get_penattr(const VTermState *state, VTermAttr attr, VTermValue
     val->boolean = state->pen.reverse;
     return 1;
 
+  case VTERM_ATTR_CONCEAL:
+    val->boolean = state->pen.conceal;
+    return 1;
+
   case VTERM_ATTR_STRIKE:
     val->boolean = state->pen.strike;
     return 1;

src/libvterm/src/screen.c

@@ -21,6 +21,7 @@ typedef struct
   unsigned int italic    : 1;
   unsigned int blink     : 1;
   unsigned int reverse   : 1;
+  unsigned int conceal   : 1;
   unsigned int strike    : 1;
   unsigned int font      : 4; // 0 to 9
 
@@ -37,8 +38,6 @@ typedef struct
   ScreenPen pen;
 } ScreenCell;
 
-static int vterm_screen_set_cell(VTermScreen *screen, VTermPos pos, const VTermScreenCell *cell);
-
 struct VTermScreen
 {
   VTerm *vt;
@@ -69,6 +68,12 @@ struct VTermScreen
   ScreenPen pen;
 };
 
+static void clearcell(const VTermScreen *screen, ScreenCell *cell)
+{
+  cell->chars[0] = 0;
+  cell->pen = screen->pen;
+}
+
 static ScreenCell *getcell(const VTermScreen *screen, int row, int col)
 {
   if(row < 0 || row >= screen->rows)
@@ -80,28 +85,18 @@ static ScreenCell *getcell(const VTermScreen *screen, int row, int col)
   return screen->buffer + (screen->cols * row) + col;
 }
 
-static ScreenCell *realloc_buffer(VTermScreen *screen, ScreenCell *buffer, int new_rows, int new_cols)
+static ScreenCell *alloc_buffer(VTermScreen *screen, int rows, int cols)
 {
-  ScreenCell *new_buffer = vterm_allocator_malloc(screen->vt, sizeof(ScreenCell) * new_rows * new_cols);
-  int row, col;
+  ScreenCell *new_buffer = vterm_allocator_malloc(screen->vt, sizeof(ScreenCell) * rows * cols);
+  int row;
+  int col;
 
-  if (new_buffer == NULL)
-    return NULL;
-  for(row = 0; row < new_rows; row++) {
-    for(col = 0; col < new_cols; col++) {
-      ScreenCell *new_cell = new_buffer + row*new_cols + col;
-
-      if(buffer && row < screen->rows && col < screen->cols)
-        *new_cell = buffer[row * screen->cols + col];
-      else {
-        new_cell->chars[0] = 0;
-        new_cell->pen = screen->pen;
-      }
+  for(row = 0; row < rows; row++) {
+    for(col = 0; col < cols; col++) {
+      clearcell(screen, &new_buffer[row * cols + col]);
     }
   }
 
-  vterm_allocator_free(screen->vt, buffer);
-
   return new_buffer;
 }
 
@@ -207,21 +202,27 @@ static int putglyph(VTermGlyphInfo *info, VTermPos pos, void *user)
   return 1;
 }
 
+static void sb_pushline_from_row(VTermScreen *screen, int row)
+{
+  VTermPos pos;
+  pos.row = row;
+  for(pos.col = 0; pos.col < screen->cols; pos.col++)
+    vterm_screen_get_cell(screen, pos, screen->sb_buffer + pos.col);
+
+  (screen->callbacks->sb_pushline)(screen->cols, screen->sb_buffer, screen->cbdata);
+}
+
 static int moverect_internal(VTermRect dest, VTermRect src, void *user)
 {
   VTermScreen *screen = user;
 
   if(screen->callbacks && screen->callbacks->sb_pushline &&
-     dest.start_row == 0 && dest.start_col == 0 &&  // starts top-left corner
-     dest.end_col == screen->cols &&                // full width
-     screen->buffer == screen->buffers[0]) {        // not altscreen
-    VTermPos pos;
-    for(pos.row = 0; pos.row < src.start_row; pos.row++) {
-      for(pos.col = 0; pos.col < screen->cols; pos.col++)
-        (void)vterm_screen_get_cell(screen, pos, screen->sb_buffer + pos.col);
-
-      (screen->callbacks->sb_pushline)(screen->cols, screen->sb_buffer, screen->cbdata);
-    }
+     dest.start_row == 0 && dest.start_col == 0 &&        // starts top-left corner
+     dest.end_col == screen->cols &&                      // full width
+     screen->buffer == screen->buffers[BUFIDX_PRIMARY]) { // not altscreen
+    int row;
+    for(row = 0; row < src.start_row; row++)
+      sb_pushline_from_row(screen, row);
   }
 
   {
@@ -420,6 +421,9 @@ static int setpenattr(VTermAttr attr, VTermValue *val, void *user)
   case VTERM_ATTR_REVERSE:
     screen->pen.reverse = val->boolean;
     return 1;
+  case VTERM_ATTR_CONCEAL:
+    screen->pen.conceal = val->boolean;
+    return 1;
   case VTERM_ATTR_STRIKE:
     screen->pen.strike = val->boolean;
     return 1;
@@ -446,10 +450,10 @@ static int settermprop(VTermProp prop, VTermValue *val, void *user)
 
   switch(prop) {
   case VTERM_PROP_ALTSCREEN:
-    if(val->boolean && !screen->buffers[1])
+    if(val->boolean && !screen->buffers[BUFIDX_ALTSCREEN])
       return 0;
 
-    screen->buffer = val->boolean ? screen->buffers[1] : screen->buffers[0];
+    screen->buffer = val->boolean ? screen->buffers[BUFIDX_ALTSCREEN] : screen->buffers[BUFIDX_PRIMARY];
     // only send a damage event on disable; because during enable there's an
     // erase that sends a damage anyway
     if(!val->boolean)
@@ -479,95 +483,146 @@ static int bell(void *user)
   return 0;
 }
 
-static int resize(int new_rows, int new_cols, VTermPos *delta, void *user)
+static void resize_buffer(VTermScreen *screen, int bufidx, int new_rows, int new_cols, int active, VTermStateFields *statefields)
 {
-  VTermScreen *screen = user;
-
-  int is_altscreen = (screen->buffers[1] && screen->buffer == screen->buffers[1]);
-
   int old_rows = screen->rows;
   int old_cols = screen->cols;
-  int first_blank_row;
 
-  if(!is_altscreen && new_rows < old_rows) {
-    // Fewer rows - determine if we're going to scroll at all, and if so, push
-    // those lines to scrollback
-    VTermPos pos = { 0, 0 };
-    VTermPos cursor = screen->state->pos;
-    // Find the first blank row after the cursor.
-    for(pos.row = old_rows - 1; pos.row >= new_rows; pos.row--)
-      if(!vterm_screen_is_eol(screen, pos) || cursor.row == pos.row)
-        break;
+  ScreenCell *old_buffer = screen->buffers[bufidx];
+  ScreenCell *new_buffer = vterm_allocator_malloc(screen->vt, sizeof(ScreenCell) * new_rows * new_cols);
 
-    first_blank_row = pos.row + 1;
-    if(first_blank_row > new_rows) {
-      VTermRect rect = {0,0,0,0};
-      rect.end_row   = old_rows;
-      rect.end_col   = old_cols;
-      scrollrect(rect, first_blank_row - new_rows, 0, user);
-      vterm_screen_flush_damage(screen);
+  /* Find the final row of old buffer content */
+  int old_row = old_rows - 1;
+  int new_row = new_rows - 1;
+  int col;
 
-      delta->row -= first_blank_row - new_rows;
+  while(new_row >= 0 && old_row >= 0) {
+    for(col = 0; col < old_cols && col < new_cols; col++)
+      new_buffer[new_row * new_cols + col] = old_buffer[old_row * old_cols + col];
+    for( ; col < new_cols; col++)
+      clearcell(screen, &new_buffer[new_row * new_cols + col]);
+
+    old_row--;
+    new_row--;
+
+    if(new_row < 0 && old_row >= 0 &&
+        new_buffer[(new_rows - 1) * new_cols].chars[0] == 0 &&
+        (!active || statefields->pos.row < (new_rows - 1))) {
+      int moverows = new_rows - 1;
+      memmove(&new_buffer[1 * new_cols], &new_buffer[0], moverows * new_cols * sizeof(ScreenCell));
+
+      new_row++;
     }
   }
 
-  screen->buffers[0] = realloc_buffer(screen, screen->buffers[0], new_rows, new_cols);
-  if(screen->buffers[1])
-    screen->buffers[1] = realloc_buffer(screen, screen->buffers[1], new_rows, new_cols);
+  if(old_row >= 0 && bufidx == BUFIDX_PRIMARY) {
+    /* Push spare lines to scrollback buffer */
+    int row;
+    for(row = 0; row <= old_row; row++)
+      sb_pushline_from_row(screen, row);
+    if(active)
+      statefields->pos.row -= (old_row + 1);
+  }
+  if(new_row >= 0 && bufidx == BUFIDX_PRIMARY &&
+      screen->callbacks && screen->callbacks->sb_popline) {
+    /* Try to backfill rows by popping scrollback buffer */
+    while(new_row >= 0) {
+      VTermPos pos;
+      if(!(screen->callbacks->sb_popline(old_cols, screen->sb_buffer, screen->cbdata)))
+        break;
 
-  screen->buffer = is_altscreen ? screen->buffers[1] : screen->buffers[0];
+      pos.row = new_row;
+      for(pos.col = 0; pos.col < old_cols && pos.col < new_cols; pos.col += screen->sb_buffer[pos.col].width) {
+        VTermScreenCell *src = &screen->sb_buffer[pos.col];
+        ScreenCell *dst = &new_buffer[pos.row * new_cols + pos.col];
+	int i;
+
+        for(i = 0; i < VTERM_MAX_CHARS_PER_CELL; i++) {
+          dst->chars[i] = src->chars[i];
+          if(!src->chars[i])
+            break;
+        }
+
+        dst->pen.bold      = src->attrs.bold;
+        dst->pen.underline = src->attrs.underline;
+        dst->pen.italic    = src->attrs.italic;
+        dst->pen.blink     = src->attrs.blink;
+        dst->pen.reverse   = src->attrs.reverse ^ screen->global_reverse;
+        dst->pen.conceal   = src->attrs.conceal;
+        dst->pen.strike    = src->attrs.strike;
+        dst->pen.font      = src->attrs.font;
+
+        dst->pen.fg = src->fg;
+        dst->pen.bg = src->bg;
+
+        if(src->width == 2 && pos.col < (new_cols-1))
+          (dst + 1)->chars[0] = (uint32_t) -1;
+      }
+      for( ; pos.col < new_cols; pos.col++)
+        clearcell(screen, &new_buffer[pos.row * new_cols + pos.col]);
+      new_row--;
+
+      if(active)
+        statefields->pos.row++;
+    }
+  }
+
+  if(new_row >= 0) {
+    /* Scroll new rows back up to the top and fill in blanks at the bottom */
+    int moverows = new_rows - new_row - 1;
+    memmove(&new_buffer[0], &new_buffer[(new_row + 1) * new_cols], moverows * new_cols * sizeof(ScreenCell));
+
+    for(new_row = moverows; new_row < new_rows; new_row++)
+    {
+      for(col = 0; col < new_cols; col++)
+        clearcell(screen, &new_buffer[new_row * new_cols + col]);
+    }
+  }
+
+  vterm_allocator_free(screen->vt, old_buffer);
+  screen->buffers[bufidx] = new_buffer;
+
+  return;
+
+  /* REFLOW TODO:
+   *   Handle delta. Probably needs to be a full cursorpos that we edit
+   */
+}
+
+static int resize(int new_rows, int new_cols, VTermStateFields *fields, void *user)
+{
+  VTermScreen *screen = user;
+
+  int altscreen_active = (screen->buffers[BUFIDX_ALTSCREEN] && screen->buffer == screen->buffers[BUFIDX_ALTSCREEN]);
+
+  int old_cols = screen->cols;
+
+  if(new_cols > old_cols) {
+    /* Ensure that ->sb_buffer is large enough for a new or and old row */
+    if(screen->sb_buffer)
+      vterm_allocator_free(screen->vt, screen->sb_buffer);
+
+    screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols);
+  }
+
+  resize_buffer(screen, 0, new_rows, new_cols, !altscreen_active, fields);
+  if(screen->buffers[BUFIDX_ALTSCREEN])
+    resize_buffer(screen, 1, new_rows, new_cols, altscreen_active, fields);
+
+  screen->buffer = altscreen_active ? screen->buffers[BUFIDX_ALTSCREEN] : screen->buffers[BUFIDX_PRIMARY];
 
   screen->rows = new_rows;
   screen->cols = new_cols;
 
-  vterm_allocator_free(screen->vt, screen->sb_buffer);
+  if(new_cols <= old_cols) {
+    if(screen->sb_buffer)
+      vterm_allocator_free(screen->vt, screen->sb_buffer);
 
-  screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols);
-
-  if(new_cols > old_cols) {
-    VTermRect rect;
-    rect.start_row = 0;
-    rect.end_row   = old_rows;
-    rect.start_col = old_cols;
-    rect.end_col   = new_cols;
-    damagerect(screen, rect);
+    screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols);
   }
 
-  if(new_rows > old_rows) {
-    if(!is_altscreen && screen->callbacks && screen->callbacks->sb_popline) {
-      int rows = new_rows - old_rows;
-      while(rows) {
-        VTermRect rect = {0,0,0,0};
-        VTermPos pos = { 0, 0 };
-        if(!(screen->callbacks->sb_popline(screen->cols, screen->sb_buffer, screen->cbdata)))
-          break;
-
-	rect.end_row   = screen->rows;
-	rect.end_col   = screen->cols;
-        scrollrect(rect, -1, 0, user);
-
-        for(pos.col = 0; pos.col < screen->cols; pos.col += screen->sb_buffer[pos.col].width)
-          vterm_screen_set_cell(screen, pos, screen->sb_buffer + pos.col);
-
-        rect.end_row = 1;
-        damagerect(screen, rect);
-
-        vterm_screen_flush_damage(screen);
-
-        rows--;
-        delta->row++;
-      }
-    }
-
-    {
-      VTermRect rect;
-      rect.start_row = old_rows;
-      rect.end_row   = new_rows;
-      rect.start_col = 0;
-      rect.end_col   = new_cols;
-      damagerect(screen, rect);
-    }
-  }
+  /* TODO: Maaaaybe we can optimise this if there's no reflow happening */
+  damagescreen(screen);
 
   if(screen->callbacks && screen->callbacks->resize)
     return (*screen->callbacks->resize)(new_rows, new_cols, screen->cbdata);
@@ -651,8 +706,10 @@ static VTermScreen *screen_new(VTerm *vt)
   screen->callbacks = NULL;
   screen->cbdata    = NULL;
 
-  screen->buffers[0] = realloc_buffer(screen, NULL, rows, cols);
-  screen->buffer = screen->buffers[0];
+  screen->buffers[BUFIDX_PRIMARY] = alloc_buffer(screen, rows, cols);
+
+  screen->buffer = screen->buffers[BUFIDX_PRIMARY];
+
   screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * cols);
   if (screen->buffer == NULL || screen->sb_buffer == NULL)
   {
@@ -667,8 +724,10 @@ static VTermScreen *screen_new(VTerm *vt)
 
 INTERNAL void vterm_screen_free(VTermScreen *screen)
 {
-  vterm_allocator_free(screen->vt, screen->buffers[0]);
-  vterm_allocator_free(screen->vt, screen->buffers[1]);
+  vterm_allocator_free(screen->vt, screen->buffers[BUFIDX_PRIMARY]);
+  if(screen->buffers[BUFIDX_ALTSCREEN])
+    vterm_allocator_free(screen->vt, screen->buffers[BUFIDX_ALTSCREEN]);
+
   vterm_allocator_free(screen->vt, screen->sb_buffer);
   vterm_allocator_free(screen->vt, screen);
 }
@@ -752,7 +811,7 @@ int vterm_screen_get_cell(const VTermScreen *screen, VTermPos pos, VTermScreenCe
   if(!intcell)
     return 0;
 
-  for(i = 0; ; i++) {
+  for(i = 0; i < VTERM_MAX_CHARS_PER_CELL; i++) {
     cell->chars[i] = intcell->chars[i];
     if(!intcell->chars[i])
       break;
@@ -763,6 +822,7 @@ int vterm_screen_get_cell(const VTermScreen *screen, VTermPos pos, VTermScreenCe
   cell->attrs.italic    = intcell->pen.italic;
   cell->attrs.blink     = intcell->pen.blink;
   cell->attrs.reverse   = intcell->pen.reverse ^ screen->global_reverse;
+  cell->attrs.conceal   = intcell->pen.conceal;
   cell->attrs.strike    = intcell->pen.strike;
   cell->attrs.font      = intcell->pen.font;
 
@@ -798,41 +858,6 @@ int vterm_screen_get_cell(const VTermScreen *screen, VTermPos pos, VTermScreenCe
   return 1;
 }
 
-/*
- * Copy external to internal representation of a screen cell
- * static because it's only used internally for sb_popline during resize
- */
-static int vterm_screen_set_cell(VTermScreen *screen, VTermPos pos, const VTermScreenCell *cell)
-{
-  ScreenCell *intcell = getcell(screen, pos.row, pos.col);
-  int i;
-
-  if(!intcell)
-    return 0;
-
-  for(i = 0; ; i++) {
-    intcell->chars[i] = cell->chars[i];
-    if(!cell->chars[i])
-      break;
-  }
-
-  intcell->pen.bold      = cell->attrs.bold;
-  intcell->pen.underline = cell->attrs.underline;
-  intcell->pen.italic    = cell->attrs.italic;
-  intcell->pen.blink     = cell->attrs.blink;
-  intcell->pen.reverse   = cell->attrs.reverse ^ screen->global_reverse;
-  intcell->pen.strike    = cell->attrs.strike;
-  intcell->pen.font      = cell->attrs.font;
-
-  intcell->pen.fg = cell->fg;
-  intcell->pen.bg = cell->bg;
-
-  if(cell->width == 2)
-    getcell(screen, pos.row, pos.col + 1)->chars[0] = (uint32_t)-1;
-
-  return 1;
-}
-
 int vterm_screen_is_eol(const VTermScreen *screen, VTermPos pos)
 {
   // This cell is EOL if this and every cell to the right is black
@@ -854,12 +879,11 @@ VTermScreen *vterm_obtain_screen(VTerm *vt)
 
 void vterm_screen_enable_altscreen(VTermScreen *screen, int altscreen)
 {
-
-  if(!screen->buffers[1] && altscreen) {
+  if(!screen->buffers[BUFIDX_ALTSCREEN] && altscreen) {
     int rows, cols;
     vterm_get_size(screen->vt, &rows, &cols);
 
-    screen->buffers[1] = realloc_buffer(screen, NULL, rows, cols);
+    screen->buffers[BUFIDX_ALTSCREEN] = alloc_buffer(screen, rows, cols);
   }
 }
 
@@ -874,7 +898,7 @@ void *vterm_screen_get_cbdata(VTermScreen *screen)
   return screen->cbdata;
 }
 
-void vterm_screen_set_unrecognised_fallbacks(VTermScreen *screen, const VTermParserCallbacks *fallbacks, void *user)
+void vterm_screen_set_unrecognised_fallbacks(VTermScreen *screen, const VTermStateFallbacks *fallbacks, void *user)
 {
   vterm_state_set_unrecognised_fallbacks(screen->state, fallbacks, user);
 }
@@ -919,13 +943,15 @@ static int attrs_differ(VTermAttrMask attrs, ScreenCell *a, ScreenCell *b)
     return 1;
   if((attrs & VTERM_ATTR_REVERSE_MASK)    && (a->pen.reverse != b->pen.reverse))
     return 1;
+  if((attrs & VTERM_ATTR_CONCEAL_MASK)    && (a->pen.conceal != b->pen.conceal))
+    return 1;
   if((attrs & VTERM_ATTR_STRIKE_MASK)     && (a->pen.strike != b->pen.strike))
     return 1;
   if((attrs & VTERM_ATTR_FONT_MASK)       && (a->pen.font != b->pen.font))
     return 1;
-  if((attrs & VTERM_ATTR_FOREGROUND_MASK) && !vterm_color_equal(a->pen.fg, b->pen.fg))
+  if((attrs & VTERM_ATTR_FOREGROUND_MASK) && !vterm_color_is_equal(&a->pen.fg, &b->pen.fg))
     return 1;
-  if((attrs & VTERM_ATTR_BACKGROUND_MASK) && !vterm_color_equal(a->pen.bg, b->pen.bg))
+  if((attrs & VTERM_ATTR_BACKGROUND_MASK) && !vterm_color_is_equal(&a->pen.bg, &b->pen.bg))
     return 1;
 
   return 0;
@@ -958,3 +984,8 @@ int vterm_screen_get_attrs_extent(const VTermScreen *screen, VTermRect *extent,
 
   return 1;
 }
+
+void vterm_screen_convert_color_to_rgb(const VTermScreen *screen, VTermColor *col)
+{
+  vterm_state_convert_color_to_rgb(screen->state, col);
+}

src/libvterm/src/state.c

@@ -44,6 +44,15 @@ static void updatecursor(VTermState *state, VTermPos *oldpos, int cancel_phantom
 
 static void erase(VTermState *state, VTermRect rect, int selective)
 {
+  if(rect.end_col == state->cols) {
+    int row;
+    /* If we're erasing the final cells of any lines, cancel the continuation
+     * marker on the subsequent line
+     */
+    for(row = rect.start_row + 1; row < rect.end_row + 1 && row < state->rows; row++)
+      state->lineinfo[row].continuation = 0;
+  }
+
   if(state->callbacks && state->callbacks->erase)
     if((*state->callbacks->erase)(rect, selective, state->cbdata))
       return;
@@ -73,13 +82,29 @@ static VTermState *vterm_state_new(VTerm *vt)
 
   state->bold_is_highbright = 0;
 
+  state->combine_chars_size = 16;
+  state->combine_chars = vterm_allocator_malloc(state->vt, state->combine_chars_size * sizeof(state->combine_chars[0]));
+
+  state->tabstops = vterm_allocator_malloc(state->vt, (state->cols + 7) / 8);
+
+  state->lineinfos[BUFIDX_PRIMARY]   = vterm_allocator_malloc(state->vt, state->rows * sizeof(VTermLineInfo));
+  /* TODO: Make an 'enable' function */
+  state->lineinfos[BUFIDX_ALTSCREEN] = vterm_allocator_malloc(state->vt, state->rows * sizeof(VTermLineInfo));
+  state->lineinfo = state->lineinfos[BUFIDX_PRIMARY];
+
+  state->encoding_utf8.enc = vterm_lookup_encoding(ENC_UTF8, 'u');
+  if(state->encoding_utf8.enc->init)
+    (*state->encoding_utf8.enc->init)(state->encoding_utf8.enc, state->encoding_utf8.data);
+
   return state;
 }
 
 INTERNAL void vterm_state_free(VTermState *state)
 {
   vterm_allocator_free(state->vt, state->tabstops);
-  vterm_allocator_free(state->vt, state->lineinfo);
+  vterm_allocator_free(state->vt, state->lineinfos[BUFIDX_PRIMARY]);
+  if(state->lineinfos[BUFIDX_ALTSCREEN])
+    vterm_allocator_free(state->vt, state->lineinfos[BUFIDX_ALTSCREEN]);
   vterm_allocator_free(state->vt, state->combine_chars);
   vterm_allocator_free(state->vt, state);
 }
@@ -106,15 +131,23 @@ static void scroll(VTermState *state, VTermRect rect, int downward, int rightwar
   // Update lineinfo if full line
   if(rect.start_col == 0 && rect.end_col == state->cols && rightward == 0) {
     int height = rect.end_row - rect.start_row - abs(downward);
+    int row;
+    VTermLineInfo zeroLineInfo = { 0 };
 
-    if(downward > 0)
+    if(downward > 0) {
       memmove(state->lineinfo + rect.start_row,
               state->lineinfo + rect.start_row + downward,
               height * sizeof(state->lineinfo[0]));
-    else
+      for(row = rect.end_row - downward; row < rect.end_row; row++)
+        state->lineinfo[row] = zeroLineInfo;
+    }
+    else {
       memmove(state->lineinfo + rect.start_row - downward,
               state->lineinfo + rect.start_row,
               height * sizeof(state->lineinfo[0]));
+      for(row = rect.start_row; row < rect.start_row - downward; row++)
+        state->lineinfo[row] = zeroLineInfo;
+    }
   }
 
   if(state->callbacks && state->callbacks->scrollrect)
@@ -367,6 +400,7 @@ static int on_text(const char bytes[], size_t len, void *user)
       linefeed(state);
       state->pos.col = 0;
       state->at_phantom = 0;
+      state->lineinfo[state->pos.row].continuation = 1;
     }
 
     if(state->mode.insert) {
@@ -548,19 +582,12 @@ static int settermprop_int(VTermState *state, VTermProp prop, int v)
   return vterm_state_set_termprop(state, prop, &val);
 }
 
-static int settermprop_string(VTermState *state, VTermProp prop, const char *str, size_t len)
+static int settermprop_string(VTermState *state, VTermProp prop, VTermStringFragment frag)
 {
-  char *strvalue;
-  int r;
   VTermValue val;
-  strvalue = vterm_allocator_malloc(state->vt, (len+1) * sizeof(char));
-  strncpy(strvalue, str, len);
-  strvalue[len] = 0;
 
-  val.string = strvalue;
-  r = vterm_state_set_termprop(state, prop, &val);
-  vterm_allocator_free(state->vt, strvalue);
-  return r;
+  val.string = frag;
+  return vterm_state_set_termprop(state, prop, &val);
 }
 
 static void savecursor(VTermState *state, int save)
@@ -922,6 +949,7 @@ static int on_csi(const char *leader, const long args[], int argcount, const cha
   VTermState *state = user;
   int leader_byte = 0;
   int intermed_byte = 0;
+  int cancel_phantom = 1;
   VTermPos oldpos = state->pos;
   int handled = 1;
 
@@ -1237,6 +1265,24 @@ static int on_csi(const char *leader, const long args[], int argcount, const cha
     state->at_phantom = 0;
     break;
 
+  case 0x62: { // REP - ECMA-48 8.3.103
+    const int row_width = THISROWWIDTH(state);
+    count = CSI_ARG_COUNT(args[0]);
+    col = state->pos.col + count;
+    UBOUND(col, row_width);
+    while (state->pos.col < col) {
+      putglyph(state, state->combine_chars, state->combine_width, state->pos);
+      state->pos.col += state->combine_width;
+    }
+    if (state->pos.col + state->combine_width >= row_width) {
+      if (state->mode.autowrap) {
+        state->at_phantom = 1;
+        cancel_phantom = 0;
+      }
+    }
+    break;
+  }
+
   case 0x63: // DA - ECMA-48 8.3.24
     val = CSI_ARG_OR(args[0], 0);
     if(val == 0)
@@ -1432,6 +1478,14 @@ static int on_csi(const char *leader, const long args[], int argcount, const cha
       state->scrollregion_bottom = -1;
     }
 
+    // Setting the scrolling region restores the cursor to the home position
+    state->pos.row = 0;
+    state->pos.col = 0;
+    if(state->mode.origin) {
+      state->pos.row += state->scrollregion_top;
+      state->pos.col += SCROLLREGION_LEFT(state);
+    }
+
     break;
 
   case 0x73: // DECSLRM - DEC custom
@@ -1453,6 +1507,14 @@ static int on_csi(const char *leader, const long args[], int argcount, const cha
       state->scrollregion_right = -1;
     }
 
+    // Setting the scrolling region restores the cursor to the home position
+    state->pos.row = 0;
+    state->pos.col = 0;
+    if(state->mode.origin) {
+      state->pos.row += state->scrollregion_top;
+      state->pos.col += SCROLLREGION_LEFT(state);
+    }
+
     break;
 
   case 0x74:
@@ -1523,7 +1585,7 @@ static int on_csi(const char *leader, const long args[], int argcount, const cha
     UBOUND(state->pos.col, THISROWWIDTH(state)-1);
   }
 
-  updatecursor(state, &oldpos, 1);
+  updatecursor(state, &oldpos, cancel_phantom);
 
 #ifdef DEBUG
   if(state->pos.row < 0 || state->pos.row >= state->rows ||
@@ -1549,83 +1611,119 @@ static int on_csi(const char *leader, const long args[], int argcount, const cha
   return 1;
 }
 
-static int on_osc(const char *command, size_t cmdlen, void *user)
+static int on_osc(int command, VTermStringFragment frag, void *user)
 {
   VTermState *state = user;
 
-  if(cmdlen < 2)
-    return 0;
-
-  if(strneq(command, "0;", 2)) {
-    settermprop_string(state, VTERM_PROP_ICONNAME, command + 2, cmdlen - 2);
-    settermprop_string(state, VTERM_PROP_TITLE, command + 2, cmdlen - 2);
-    return 1;
-  }
-  else if(strneq(command, "1;", 2)) {
-    settermprop_string(state, VTERM_PROP_ICONNAME, command + 2, cmdlen - 2);
-    return 1;
-  }
-  else if(strneq(command, "2;", 2)) {
-    settermprop_string(state, VTERM_PROP_TITLE, command + 2, cmdlen - 2);
-    return 1;
-  }
-  else if(strneq(command, "10;", 3)) {
-    // request foreground color: <Esc>]10;?<0x07>
-    int red = state->default_fg.red;
-    int blue = state->default_fg.blue;
-    int green = state->default_fg.green;
-    vterm_push_output_sprintf_ctrl(state->vt, C1_OSC, "10;rgb:%02x%02x/%02x%02x/%02x%02x\x07", red, red, green, green, blue, blue);
-    return 1;
-  }
-  else if(strneq(command, "11;", 3)) {
-    // request background color: <Esc>]11;?<0x07>
-    int red = state->default_bg.red;
-    int blue = state->default_bg.blue;
-    int green = state->default_bg.green;
-    vterm_push_output_sprintf_ctrl(state->vt, C1_OSC, "11;rgb:%02x%02x/%02x%02x/%02x%02x\x07", red, red, green, green, blue, blue);
-    return 1;
-  }
-  else if(strneq(command, "12;", 3)) {
-    settermprop_string(state, VTERM_PROP_CURSORCOLOR, command + 3, cmdlen - 3);
-    return 1;
-  }
-  else if(state->fallbacks && state->fallbacks->osc)
-    if((*state->fallbacks->osc)(command, cmdlen, state->fbdata))
+  switch(command) {
+    case 0:
+      settermprop_string(state, VTERM_PROP_ICONNAME, frag);
+      settermprop_string(state, VTERM_PROP_TITLE, frag);
       return 1;
 
+    case 1:
+      settermprop_string(state, VTERM_PROP_ICONNAME, frag);
+      return 1;
+
+    case 2:
+      settermprop_string(state, VTERM_PROP_TITLE, frag);
+      return 1;
+
+    case 10:
+      {
+        // request foreground color: <Esc>]10;?<0x07>
+        int red = state->default_fg.red;
+        int blue = state->default_fg.blue;
+        int green = state->default_fg.green;
+        vterm_push_output_sprintf_ctrl(state->vt, C1_OSC, "10;rgb:%02x%02x/%02x%02x/%02x%02x\x07", red, red, green, green, blue, blue);
+        return 1;
+      }
+
+    case 11:
+      {
+	// request background color: <Esc>]11;?<0x07>
+	int red = state->default_bg.red;
+	int blue = state->default_bg.blue;
+	int green = state->default_bg.green;
+	vterm_push_output_sprintf_ctrl(state->vt, C1_OSC, "11;rgb:%02x%02x/%02x%02x/%02x%02x\x07", red, red, green, green, blue, blue);
+	return 1;
+      }
+    case 12:
+      settermprop_string(state, VTERM_PROP_CURSORCOLOR, frag);
+      return 1;
+
+    default:
+      if(state->fallbacks && state->fallbacks->osc)
+        if((*state->fallbacks->osc)(command, frag, state->fbdata))
+          return 1;
+  }
+
   return 0;
 }
 
-static void request_status_string(VTermState *state, const char *command, size_t cmdlen)
+static void request_status_string(VTermState *state, VTermStringFragment frag)
 {
-  if(cmdlen == 1)
-    switch(command[0]) {
-      case 'm': // Query SGR
-        {
-          long args[20];
-          int argc = vterm_state_getpen(state, args, sizeof(args)/sizeof(args[0]));
-	  int argi;
-          vterm_push_output_sprintf_ctrl(state->vt, C1_DCS, "1$r");
-          for(argi = 0; argi < argc; argi++)
-            vterm_push_output_sprintf(state->vt,
-                argi == argc - 1             ? "%d" :
-                CSI_ARG_HAS_MORE(args[argi]) ? "%d:" :
-                                               "%d;",
-                CSI_ARG(args[argi]));
-          vterm_push_output_sprintf(state->vt, "m");
-          vterm_push_output_sprintf_ctrl(state->vt, C1_ST, "");
-        }
+  VTerm *vt = state->vt;
+
+  char *tmp = state->tmp.decrqss;
+  size_t i = 0;
+
+  if(frag.initial)
+    tmp[0] = tmp[1] = tmp[2] = tmp[3] = 0;
+
+  while(i < sizeof(state->tmp.decrqss)-1 && tmp[i])
+    i++;
+  while(i < sizeof(state->tmp.decrqss)-1 && frag.len--)
+    tmp[i++] = (frag.str++)[0];
+  tmp[i] = 0;
+
+  if(!frag.final)
+    return;
+
+  switch(tmp[0] | tmp[1]<<8 | tmp[2]<<16) {
+    case 'm': {
+      // Query SGR
+      long args[20];
+      int argc = vterm_state_getpen(state, args, sizeof(args)/sizeof(args[0]));
+      size_t cur = 0;
+      int argi;
+
+      cur += SNPRINTF(vt->tmpbuffer + cur, vt->tmpbuffer_len - cur,
+          vt->mode.ctrl8bit ? "\x90" "1$r" : ESC_S "P" "1$r"); // DCS 1$r ...
+      if(cur >= vt->tmpbuffer_len)
         return;
-      case 'r': // Query DECSTBM
-        vterm_push_output_sprintf_dcs(state->vt, "1$r%d;%dr", state->scrollregion_top+1, SCROLLREGION_BOTTOM(state));
-        return;
-      case 's': // Query DECSLRM
-        vterm_push_output_sprintf_dcs(state->vt, "1$r%d;%ds", SCROLLREGION_LEFT(state)+1, SCROLLREGION_RIGHT(state));
+
+      for(argi = 0; argi < argc; argi++) {
+        cur += SNPRINTF(vt->tmpbuffer + cur, vt->tmpbuffer_len - cur,
+            argi == argc - 1             ? "%ld" :
+            CSI_ARG_HAS_MORE(args[argi]) ? "%ld:" :
+                                           "%ld;",
+            CSI_ARG(args[argi]));
+        if(cur >= vt->tmpbuffer_len)
+          return;
+      }
+
+      cur += SNPRINTF(vt->tmpbuffer + cur, vt->tmpbuffer_len - cur,
+          vt->mode.ctrl8bit ? "m" "\x9C" : "m" ESC_S "\\"); // ... m ST
+      if(cur >= vt->tmpbuffer_len)
         return;
+
+      vterm_push_output_bytes(vt, vt->tmpbuffer, cur);
+      return;
     }
 
-  if(cmdlen == 2) {
-    if(strneq(command, " q", 2)) {
+    case 'r':
+      // Query DECSTBM
+      vterm_push_output_sprintf_dcs(vt, "1$r%d;%dr", state->scrollregion_top+1, SCROLLREGION_BOTTOM(state));
+      return;
+
+    case 's':
+      // Query DECSLRM
+      vterm_push_output_sprintf_dcs(vt, "1$r%d;%ds", SCROLLREGION_LEFT(state)+1, SCROLLREGION_RIGHT(state));
+      return;
+
+    case ' '|('q'<<8): {
+      // Query DECSCUSR
       int reply;
       switch(state->mode.cursor_shape) {
         case VTERM_PROP_CURSORSHAPE_BLOCK:     reply = 2; break;
@@ -1634,30 +1732,32 @@ static void request_status_string(VTermState *state, const char *command, size_t
       }
       if(state->mode.cursor_blink)
         reply--;
-      vterm_push_output_sprintf_dcs(state->vt, "1$r%d q", reply);
+      vterm_push_output_sprintf_dcs(vt, "1$r%d q", reply);
       return;
     }
-    else if(strneq(command, "\"q", 2)) {
-      vterm_push_output_sprintf_dcs(state->vt, "1$r%d\"q", state->protected_cell ? 1 : 2);
+
+    case '\"'|('q'<<8):
+      // Query DECSCA
+      vterm_push_output_sprintf_dcs(vt, "1$r%d\"q", state->protected_cell ? 1 : 2);
       return;
-    }
   }
 
-  vterm_push_output_sprintf_dcs(state->vt, "0$r%.s", (int)cmdlen, command);
+  vterm_push_output_sprintf_dcs(state->vt, "0$r%s", tmp);
 }
 
-static int on_dcs(const char *command, size_t cmdlen, void *user)
+static int on_dcs(const char *command, size_t commandlen, VTermStringFragment frag, void *user)
 {
   VTermState *state = user;
 
-  if(cmdlen >= 2 && strneq(command, "$q", 2)) {
-    request_status_string(state, command+2, cmdlen-2);
+  if(commandlen == 2 && strneq(command, "$q", 2)) {
+    request_status_string(state, frag);
     return 1;
   }
   else if(state->fallbacks && state->fallbacks->dcs)
-    if((*state->fallbacks->dcs)(command, cmdlen, state->fbdata))
+    if((*state->fallbacks->dcs)(command, commandlen, frag, state->fbdata))
       return 1;
 
+  DEBUG_LOG2("libvterm: Unhandled DCS %.*s\n", (int)commandlen, command);
   return 0;
 }
 
@@ -1665,7 +1765,7 @@ static int on_resize(int rows, int cols, void *user)
 {
   VTermState *state = user;
   VTermPos oldpos = state->pos;
-  VTermPos delta = { 0, 0 };
+  VTermStateFields fields;
 
   if(cols != state->cols) {
     int col;
@@ -1695,22 +1795,30 @@ static int on_resize(int rows, int cols, void *user)
   }
 
   if(rows != state->rows) {
-    int row;
-    VTermLineInfo *newlineinfo = vterm_allocator_malloc(state->vt, rows * sizeof(VTermLineInfo));
-    if (newlineinfo == NULL)
-      return 0;
+    int bufidx;
+    for(bufidx = BUFIDX_PRIMARY; bufidx <= BUFIDX_ALTSCREEN; bufidx++) {
+      int row;
+      VTermLineInfo *oldlineinfo = state->lineinfos[bufidx];
+      VTermLineInfo *newlineinfo;
+      if(!oldlineinfo)
+        continue;
 
-    for(row = 0; row < state->rows && row < rows; row++) {
-      newlineinfo[row] = state->lineinfo[row];
+      newlineinfo = vterm_allocator_malloc(state->vt, rows * sizeof(VTermLineInfo));
+
+      for(row = 0; row < state->rows && row < rows; row++) {
+        newlineinfo[row] = oldlineinfo[row];
+      }
+
+      for( ; row < rows; row++) {
+	VTermLineInfo lineInfo = {0};
+	newlineinfo[row] = lineInfo;
+      }
+
+      vterm_allocator_free(state->vt, state->lineinfos[bufidx]);
+      state->lineinfos[bufidx] = newlineinfo;
     }
 
-    for( ; row < rows; row++) {
-      newlineinfo[row].doublewidth = 0;
-      newlineinfo[row].doubleheight = 0;
-    }
-
-    vterm_allocator_free(state->vt, state->lineinfo);
-    state->lineinfo = newlineinfo;
+    state->lineinfo = state->lineinfos[state->mode.alt_screen ? BUFIDX_ALTSCREEN : BUFIDX_PRIMARY];
   }
 
   state->rows = rows;
@@ -1721,17 +1829,18 @@ static int on_resize(int rows, int cols, void *user)
   if(state->scrollregion_right > -1)
     UBOUND(state->scrollregion_right, state->cols);
 
+  fields.pos = state->pos;
+
   if(state->callbacks && state->callbacks->resize)
-    (*state->callbacks->resize)(rows, cols, &delta, state->cbdata);
+    (*state->callbacks->resize)(rows, cols, &fields, state->cbdata);
+
+  state->pos = fields.pos;
 
   if(state->at_phantom && state->pos.col < cols-1) {
     state->at_phantom = 0;
     state->pos.col++;
   }
 
-  state->pos.row += delta.row;
-  state->pos.col += delta.col;
-
   if(state->pos.row >= rows)
     state->pos.row = rows - 1;
   if(state->pos.col >= cols)
@@ -1767,17 +1876,6 @@ VTermState *vterm_obtain_state(VTerm *vt)
     return NULL;
   vt->state = state;
 
-  state->combine_chars_size = 16;
-  state->combine_chars = vterm_allocator_malloc(state->vt, state->combine_chars_size * sizeof(state->combine_chars[0]));
-
-  state->tabstops = vterm_allocator_malloc(state->vt, (state->cols + 7) / 8);
-
-  state->lineinfo = vterm_allocator_malloc(state->vt, state->rows * sizeof(VTermLineInfo));
-
-  state->encoding_utf8.enc = vterm_lookup_encoding(ENC_UTF8, 'u');
-  if(*state->encoding_utf8.enc->init != NULL)
-    (*state->encoding_utf8.enc->init)(state->encoding_utf8.enc, state->encoding_utf8.data);
-
   vterm_parser_set_callbacks(vt, &parser_callbacks, state);
 
   return state;
@@ -1895,7 +1993,7 @@ void *vterm_state_get_cbdata(VTermState *state)
   return state->cbdata;
 }
 
-void vterm_state_set_unrecognised_fallbacks(VTermState *state, const VTermParserCallbacks *fallbacks, void *user)
+void vterm_state_set_unrecognised_fallbacks(VTermState *state, const VTermStateFallbacks *fallbacks, void *user)
 {
   if(fallbacks) {
     state->fallbacks = fallbacks;
@@ -1940,6 +2038,7 @@ int vterm_state_set_termprop(VTermState *state, VTermProp prop, VTermValue *val)
     return 1;
   case VTERM_PROP_ALTSCREEN:
     state->mode.alt_screen = val->boolean;
+    state->lineinfo = state->lineinfos[state->mode.alt_screen ? BUFIDX_ALTSCREEN : BUFIDX_PRIMARY];
     if(state->mode.alt_screen) {
       VTermRect rect = {0, 0, 0, 0};
       rect.end_row = state->rows;

src/libvterm/src/unicode.c

@@ -591,8 +591,15 @@ vterm_get_special_pty_type_placeholder(void)
 // ################################
 // ### The rest added by Paul Evans
 
+static const struct interval fullwidth[] = {
+#include "fullwidth.inc"
+};
+
 INTERNAL int vterm_unicode_width(uint32_t codepoint)
 {
+  if(bisearch(codepoint, fullwidth, sizeof(fullwidth) / sizeof(fullwidth[0]) - 1))
+    return 2;
+
   return WCWIDTH_FUNCTION(codepoint);
 }
 

src/libvterm/src/vterm.c

@@ -55,21 +55,22 @@ VTerm *vterm_new_with_allocator(int rows, int cols, VTermAllocatorFunctions *fun
   vt->parser.callbacks = NULL;
   vt->parser.cbdata    = NULL;
 
-  vt->parser.strbuffer_len = 500; // should be able to hold an OSC string
-  vt->parser.strbuffer_cur = 0;
-  vt->parser.strbuffer = vterm_allocator_malloc(vt, vt->parser.strbuffer_len);
-  if (vt->parser.strbuffer == NULL)
-  {
-    vterm_allocator_free(vt, vt);
-    return NULL;
-  }
+  vt->outfunc = NULL;
+  vt->outdata = NULL;
 
   vt->outbuffer_len = 200;
   vt->outbuffer_cur = 0;
   vt->outbuffer = vterm_allocator_malloc(vt, vt->outbuffer_len);
-  if (vt->outbuffer == NULL)
+
+  vt->tmpbuffer_len = 64;
+  vt->tmpbuffer = vterm_allocator_malloc(vt, vt->tmpbuffer_len);
+
+  if (vt->tmpbuffer == NULL
+      || vt->outbuffer == NULL
+      || vt->tmpbuffer == NULL)
   {
-    vterm_allocator_free(vt, vt->parser.strbuffer);
+    vterm_allocator_free(vt, vt->outbuffer);
+    vterm_allocator_free(vt, vt->tmpbuffer);
     vterm_allocator_free(vt, vt);
     return NULL;
   }
@@ -85,8 +86,8 @@ void vterm_free(VTerm *vt)
   if(vt->state)
     vterm_state_free(vt->state);
 
-  vterm_allocator_free(vt, vt->parser.strbuffer);
   vterm_allocator_free(vt, vt->outbuffer);
+  vterm_allocator_free(vt, vt->tmpbuffer);
 
   vterm_allocator_free(vt, vt);
 }
@@ -132,70 +133,42 @@ void vterm_set_utf8(VTerm *vt, int is_utf8)
   vt->mode.utf8 = is_utf8;
 }
 
+void vterm_output_set_callback(VTerm *vt, VTermOutputCallback *func, void *user)
+{
+  vt->outfunc = func;
+  vt->outdata = user;
+}
+
 INTERNAL void vterm_push_output_bytes(VTerm *vt, const char *bytes, size_t len)
 {
+  if(vt->outfunc) {
+    (vt->outfunc)(bytes, len, vt->outdata);
+    return;
+  }
+
   if(len > vt->outbuffer_len - vt->outbuffer_cur) {
-    DEBUG_LOG("vterm_push_output(): buffer overflow; truncating output\n");
-    len = vt->outbuffer_len - vt->outbuffer_cur;
+    DEBUG_LOG("vterm_push_output_bytes(): buffer overflow; dropping output\n");
+    return;
   }
 
   memcpy(vt->outbuffer + vt->outbuffer_cur, bytes, len);
   vt->outbuffer_cur += len;
 }
 
-static int outbuffer_is_full(VTerm *vt)
-{
-  return vt->outbuffer_cur >= vt->outbuffer_len - 1;
-}
-
-#if (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE >= 500) \
-	|| defined(_ISOC99_SOURCE) || defined(_BSD_SOURCE)
-# undef VSNPRINTF
-# define VSNPRINTF vsnprintf
-#else
-# ifdef VSNPRINTF
-// Use a provided vsnprintf() function.
-int VSNPRINTF(char *str, size_t str_m, const char *fmt, va_list ap);
-# endif
-#endif
-
-
 INTERNAL void vterm_push_output_vsprintf(VTerm *vt, const char *format, va_list args)
 {
-  int written;
+  size_t len;
 #ifndef VSNPRINTF
   // When vsnprintf() is not available (C90) fall back to vsprintf().
   char buffer[1024]; // 1Kbyte is enough for everybody, right?
 #endif
 
-  if(outbuffer_is_full(vt)) {
-    DEBUG_LOG("vterm_push_output(): buffer overflow; truncating output\n");
-    return;
-  }
-
 #ifdef VSNPRINTF
-  written = VSNPRINTF(vt->outbuffer + vt->outbuffer_cur,
-      vt->outbuffer_len - vt->outbuffer_cur,
-      format, args);
-
-  if(written == (int)(vt->outbuffer_len - vt->outbuffer_cur)) {
-    // output was truncated
-    vt->outbuffer_cur = vt->outbuffer_len - 1;
-  }
-  else
-    vt->outbuffer_cur += written;
+  len = VSNPRINTF(vt->tmpbuffer, vt->tmpbuffer_len, format, args);
+  vterm_push_output_bytes(vt, vt->tmpbuffer, len);
 #else
-  written = vsprintf(buffer, format, args);
-
-  if(written >= (int)(vt->outbuffer_len - vt->outbuffer_cur - 1)) {
-    // output was truncated
-    written = vt->outbuffer_len - vt->outbuffer_cur - 1;
-  }
-  if (written > 0)
-  {
-    strncpy(vt->outbuffer + vt->outbuffer_cur, buffer, written + 1);
-    vt->outbuffer_cur += written;
-  }
+  len = vsprintf(buffer, format, args);
+  vterm_push_output_bytes(vt, buffer, len);
 #endif
 }
 
@@ -209,40 +182,45 @@ INTERNAL void vterm_push_output_sprintf(VTerm *vt, const char *format, ...)
 
 INTERNAL void vterm_push_output_sprintf_ctrl(VTerm *vt, unsigned char ctrl, const char *fmt, ...)
 {
-  size_t orig_cur = vt->outbuffer_cur;
+  size_t cur;
   va_list args;
 
   if(ctrl >= 0x80 && !vt->mode.ctrl8bit)
-    vterm_push_output_sprintf(vt, ESC_S "%c", ctrl - 0x40);
+    cur = SNPRINTF(vt->tmpbuffer, vt->tmpbuffer_len,
+        ESC_S "%c", ctrl - 0x40);
   else
-    vterm_push_output_sprintf(vt, "%c", ctrl);
+    cur = SNPRINTF(vt->tmpbuffer, vt->tmpbuffer_len,
+        "%c", ctrl);
+  if(cur >= vt->tmpbuffer_len)
+    return;
+  vterm_push_output_bytes(vt, vt->tmpbuffer, cur);
 
   va_start(args, fmt);
   vterm_push_output_vsprintf(vt, fmt, args);
   va_end(args);
-
-  if(outbuffer_is_full(vt))
-    vt->outbuffer_cur = orig_cur;
 }
 
 INTERNAL void vterm_push_output_sprintf_dcs(VTerm *vt, const char *fmt, ...)
 {
-  size_t orig_cur = vt->outbuffer_cur;
+  size_t cur;
   va_list args;
 
-  if(!vt->mode.ctrl8bit)
-    vterm_push_output_sprintf(vt, ESC_S "%c", C1_DCS - 0x40);
-  else
-    vterm_push_output_sprintf(vt, "%c", C1_DCS);
+  cur = SNPRINTF(vt->tmpbuffer, vt->tmpbuffer_len,
+      vt->mode.ctrl8bit ? "\x90" : ESC_S "P"); // DCS
+
+  if(cur >= vt->tmpbuffer_len)
+    return;
+  vterm_push_output_bytes(vt, vt->tmpbuffer, cur);
 
   va_start(args, fmt);
   vterm_push_output_vsprintf(vt, fmt, args);
   va_end(args);
 
-  vterm_push_output_sprintf_ctrl(vt, C1_ST, "");
-
-  if(outbuffer_is_full(vt))
-    vt->outbuffer_cur = orig_cur;
+  cur = SNPRINTF(vt->tmpbuffer, vt->tmpbuffer_len,
+      vt->mode.ctrl8bit ? "\x9C" : ESC_S "\\"); // ST
+  if(cur >= vt->tmpbuffer_len)
+    return;
+  vterm_push_output_bytes(vt, vt->tmpbuffer, cur);
 }
 
 size_t vterm_output_get_buffer_size(const VTerm *vt)
@@ -283,6 +261,7 @@ VTermValueType vterm_get_attr_type(VTermAttr attr)
     case VTERM_ATTR_ITALIC:     return VTERM_VALUETYPE_BOOL;
     case VTERM_ATTR_BLINK:      return VTERM_VALUETYPE_BOOL;
     case VTERM_ATTR_REVERSE:    return VTERM_VALUETYPE_BOOL;
+    case VTERM_ATTR_CONCEAL:    return VTERM_VALUETYPE_BOOL;
     case VTERM_ATTR_STRIKE:     return VTERM_VALUETYPE_BOOL;
     case VTERM_ATTR_FONT:       return VTERM_VALUETYPE_INT;
     case VTERM_ATTR_FOREGROUND: return VTERM_VALUETYPE_COLOR;
@@ -423,3 +402,20 @@ void vterm_copy_cells(VTermRect dest,
       (*copycell)(pos, srcpos, user);
     }
 }
+
+void vterm_check_version(int major, int minor)
+{
+  if(major != VTERM_VERSION_MAJOR) {
+    fprintf(stderr, "libvterm major version mismatch; %d (wants) != %d (library)\n",
+        major, VTERM_VERSION_MAJOR);
+    exit(1);
+  }
+
+  if(minor > VTERM_VERSION_MINOR) {
+    fprintf(stderr, "libvterm minor version mismatch; %d (wants) > %d (library)\n",
+        minor, VTERM_VERSION_MINOR);
+    exit(1);
+  }
+
+  // Happy
+}

src/libvterm/src/vterm_internal.h

@@ -32,6 +32,9 @@
 #define CSI_ARGS_MAX 16
 #define CSI_LEADER_MAX 16
 
+#define BUFIDX_PRIMARY   0
+#define BUFIDX_ALTSCREEN 1
+
 typedef struct VTermEncoding VTermEncoding;
 
 typedef struct {
@@ -50,19 +53,11 @@ struct VTermPen
   unsigned int italic:1;
   unsigned int blink:1;
   unsigned int reverse:1;
+  unsigned int conceal:1;
   unsigned int strike:1;
   unsigned int font:4; // To store 0-9
 };
 
-int vterm_color_equal(VTermColor a, VTermColor b);
-
-#if defined(DEFINE_INLINES) || USE_INLINE
-INLINE int vterm_color_equal(VTermColor a, VTermColor b)
-{
-  return a.red == b.red && a.green == b.green && a.blue == b.blue;
-}
-#endif
-
 struct VTermState
 {
   VTerm *vt;
@@ -70,7 +65,7 @@ struct VTermState
   const VTermStateCallbacks *callbacks;
   void *cbdata;
 
-  const VTermParserCallbacks *fallbacks;
+  const VTermStateFallbacks *fallbacks;
   void *fbdata;
 
   int rows;
@@ -92,6 +87,10 @@ struct VTermState
   // Bitvector of tab stops
   unsigned char *tabstops;
 
+  /* Primary and Altscreen; lineinfos[1] is lazily allocated as needed */
+  VTermLineInfo *lineinfos[2];
+
+  /* lineinfo will == lineinfos[0] or lineinfos[1], depending on altscreen */
   VTermLineInfo *lineinfo;
 #define ROWWIDTH(state,row) ((state)->lineinfo[(row)].doublewidth ? ((state)->cols / 2) : (state)->cols)
 #define THISROWWIDTH(state) ROWWIDTH(state, (state)->pos.row)
@@ -136,8 +135,6 @@ struct VTermState
   VTermColor default_bg;
   VTermColor colors[16]; // Store the 8 ANSI and the 8 ANSI high-brights only
 
-  int fg_index;
-  int bg_index;
   int bold_is_highbright;
 
   unsigned int protected_cell : 1;
@@ -153,15 +150,13 @@ struct VTermState
       unsigned int cursor_shape:2;
     } mode;
   } saved;
+
+  /* Temporary state for DECRQSS parsing */
+  union {
+    char decrqss[4];
+  } tmp;
 };
 
-typedef enum {
-  VTERM_PARSER_OSC,
-  VTERM_PARSER_DCS,
-
-  VTERM_N_PARSER_TYPES
-} VTermParserStringType;
-
 struct VTerm
 {
   VTermAllocatorFunctions *allocator;
@@ -181,36 +176,53 @@ struct VTerm
       CSI_LEADER,
       CSI_ARGS,
       CSI_INTERMED,
-      ESC,
-      // below here are the "string states"
-      STRING,
-      ESC_IN_STRING,
+      DCS_COMMAND,
+      /* below here are the "string states" */
+      OSC_COMMAND,
+      OSC,
+      DCS,
     } state;
 
+    unsigned int in_esc : 1;
+
     int intermedlen;
     char intermed[INTERMED_MAX];
 
-    int csi_leaderlen;
-    char csi_leader[CSI_LEADER_MAX];
+    union {
+      struct {
+        int leaderlen;
+        char leader[CSI_LEADER_MAX];
 
-    int csi_argi;
-    long csi_args[CSI_ARGS_MAX];
+        int argi;
+        long args[CSI_ARGS_MAX];
+      } csi;
+      struct {
+        int command;
+      } osc;
+      struct {
+        int commandlen;
+        char command[CSI_LEADER_MAX];
+      } dcs;
+    } v;
 
     const VTermParserCallbacks *callbacks;
     void *cbdata;
 
-    VTermParserStringType stringtype;
-    char  *strbuffer;
-    size_t strbuffer_len;
-    size_t strbuffer_cur;
+    int string_initial;
   } parser;
 
   // len == malloc()ed size; cur == number of valid bytes
 
+  VTermOutputCallback *outfunc;
+  void                *outdata;
+
   char  *outbuffer;
   size_t outbuffer_len;
   size_t outbuffer_cur;
 
+  char  *tmpbuffer;
+  size_t tmpbuffer_len;
+
   VTermState *state;
   VTermScreen *screen;
 
@@ -265,4 +277,24 @@ int vterm_unicode_is_combining(uint32_t codepoint);
 int vterm_unicode_is_ambiguous(uint32_t codepoint);
 int vterm_get_special_pty_type(void);
 
+#if (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE >= 500) \
+	|| defined(_ISOC99_SOURCE) || defined(_BSD_SOURCE)
+# undef VSNPRINTF
+# define VSNPRINTF vsnprintf
+# undef SNPRINTF
+#else
+# ifdef VSNPRINTF
+// Use a provided vsnprintf() function.
+int VSNPRINTF(char *str, size_t str_m, const char *fmt, va_list ap);
+# endif
+# ifdef SNPRINTF
+// Use a provided snprintf() function.
+int SNPRINTF(char *str, size_t str_m, const char *fmt, ...);
+# endif
+#endif
+#ifndef SNPRINTF
+# define SNPRINTF snprintf
+#endif
+
+
 #endif

src/libvterm/t/02parser.test

@@ -132,15 +132,31 @@ PUSH "\e[12\n;3X"
 
 !OSC BEL
 PUSH "\e]1;Hello\x07"
-  osc "1;Hello"
+  osc [1 "Hello"]
 
 !OSC ST (7bit)
 PUSH "\e]1;Hello\e\\"
-  osc "1;Hello"
+  osc [1 "Hello"]
 
 !OSC ST (8bit)
 PUSH "\x{9d}1;Hello\x9c"
-  osc "1;Hello"
+  osc [1 "Hello"]
+
+!OSC in parts
+PUSH "\e]52;abc"
+  osc [52 "abc"
+PUSH "def"
+  osc "def"
+PUSH "ghi\e\\"
+  osc "ghi"]
+
+!OSC BEL without semicolon 
+PUSH "\e]1234\x07"
+  osc [1234 ]
+
+!OSC ST without semicolon 
+PUSH "\e]1234\e\\"
+  osc [1234 ]
 
 !Escape cancels OSC, starts Escape
 PUSH "\e]Something\e9"
@@ -152,20 +168,21 @@ PUSH "\e]12\x{18}AB"
 
 !C0 in OSC interrupts and continues
 PUSH "\e]2;\nBye\x07"
+  osc [2 ""
   control 10
-  osc "2;Bye"
+  osc "Bye"]
 
 !DCS BEL
 PUSH "\ePHello\x07"
-  dcs "Hello"
+  dcs ["Hello"]
 
 !DCS ST (7bit)
 PUSH "\ePHello\e\\"
-  dcs "Hello"
+  dcs ["Hello"]
 
 !DCS ST (8bit)
 PUSH "\x{90}Hello\x9c"
-  dcs "Hello"
+  dcs ["Hello"]
 
 !Escape cancels DCS, starts Escape
 PUSH "\ePSomething\e9"
@@ -177,8 +194,9 @@ PUSH "\eP12\x{18}AB"
 
 !C0 in OSC interrupts and continues
 PUSH "\ePBy\ne\x07"
+  dcs ["By"
   control 10
-  dcs "Bye"
+  dcs "e"]
 
 !NUL ignored
 PUSH "\x{00}"

src/libvterm/t/10state_putglyph.test

@@ -30,6 +30,13 @@ PUSH "\xEF\xBC\x90 "
   putglyph 0xff10 2 0,0
   putglyph 0x20 1 0,2
 
+!UTF-8 emoji wide char
+# U+1F600 = 0xF0 0x9F 0x98 0x80  name: GRINNING FACE
+RESET
+PUSH "\xF0\x9F\x98\x80 "
+  putglyph 0x1f600 2 0,0
+  putglyph 0x20 1 0,2
+
 !UTF-8 combining chars
 # U+0301 = 0xCC 0x81  name: COMBINING ACUTE
 RESET

src/libvterm/t/12state_scroll.test

@@ -40,8 +40,8 @@ PUSH "\n"
   ?cursor = 20,0
 
 !Index in DECSTBM
-PUSH "\e[10H"
 PUSH "\e[9;10r"
+PUSH "\e[10H"
 PUSH "\eM"
   ?cursor = 8,0
 PUSH "\eM"
@@ -148,3 +148,9 @@ PUSH "\e[2T"
   moverect 0..23,0..80 -> 2..25,0..80
   erase 0..2,0..80
   ?cursor = 0,0
+
+!DECSTBM resets cursor position
+PUSH "\e[5;5H"
+  ?cursor = 4,4
+PUSH "\e[r"
+  ?cursor = 0,0

src/libvterm/t/18state_termprops.test

@@ -33,4 +33,10 @@ PUSH "\e[3 q"
 
 !Title
 PUSH "\e]2;Here is my title\a"
-  settermprop 4 "Here is my title"
+  settermprop 4 ["Here is my title"]
+
+!Title split write
+PUSH "\e]2;Here is"
+  settermprop 4 ["Here is"
+PUSH " another title\a"
+  settermprop 4 " another title"]

src/libvterm/t/25state_input.test

@@ -111,6 +111,18 @@ PUSH "\e[20h"
 INKEY 0 Enter
   output "\x0d\x0a"
 
+!Unmodified F1 is SS3 P
+INKEY 0 F1
+  output "\eOP"
+
+!Modified F1 is CSI P
+INKEY S F1
+  output "\e[1;2P"
+INKEY A F1
+  output "\e[1;3P"
+INKEY C F1
+  output "\e[1;5P"
+
 !Keypad in DECKPNM
 INKEY 0 KP0
   output "0"

src/libvterm/t/26state_query.test

@@ -41,10 +41,10 @@ PUSH "\e[0;93;104m"
 PUSH "\eP\$qm\e\\"
   output "\eP1\$r93;104m\e\\"
 
-!DECRQSS on SGR 256-palette colours
-PUSH "\e[0;38:5:56;48:5:78m"
-PUSH "\eP\$qm\e\\"
-  output "\eP1\$r38:5:56;48:5:78m\e\\"
+##!DECRQSS on SGR 256-palette colours
+#PUSH "\e[0;38:5:56;48:5:78m"
+#PUSH "\eP\$qm\e\\"
+#  output "\eP1\$r38:5:56;48:5:78m\e\\"
 
 !DECRQSS on SGR RGB8 colours
 PUSH "\e[0;38:2:24:68:112;48:2:13:57:101m"
@@ -59,4 +59,4 @@ PUSH "\e F"
 
 !Truncation on attempted buffer overflow
 PUSH "\e[6n" x 30
-  output "\e[10;10R" x 24
+  output "\e[10;10R" x 25

src/libvterm/t/29state_fallback.test

@@ -12,8 +12,8 @@ PUSH "\e[?15;2z"
 
 !Unrecognised OSC
 PUSH "\e]27;Something\e\\"
-  osc "27;Something"
+  osc [27 "Something"]
 
 !Unrecognised DCS
 PUSH "\ePz123\e\\"
-  dcs "z123"
+  dcs ["z123"]

src/libvterm/t/30state_pen.test

@@ -10,8 +10,8 @@ PUSH "\e[m"
   ?pen blink = off
   ?pen reverse = off
   ?pen font = 0
-  ?pen foreground = rgb(240,240,240)
-  ?pen background = rgb(0,0,0)
+  ?pen foreground = rgb(240,240,240,is_default_fg)
+  ?pen background = rgb(0,0,0,is_default_bg)
 
 !Bold
 PUSH "\e[1m"
@@ -28,6 +28,14 @@ PUSH "\e[21m"
   ?pen underline = 2
 PUSH "\e[24m"
   ?pen underline = 0
+PUSH "\e[4m\e[4:0m"
+  ?pen underline = 0
+PUSH "\e[4:1m"
+  ?pen underline = 1
+PUSH "\e[4:2m"
+  ?pen underline = 2
+PUSH "\e[4:3m"
+  ?pen underline = 3
 PUSH "\e[4m\e[m"
   ?pen underline = 0
 
@@ -67,40 +75,40 @@ PUSH "\e[11m\e[m"
 
 !Foreground
 PUSH "\e[31m"
-  ?pen foreground = rgb(224,0,0)
+  ?pen foreground = idx(1)
 PUSH "\e[32m"
-  ?pen foreground = rgb(0,224,0)
+  ?pen foreground = idx(2)
 PUSH "\e[34m"
-  ?pen foreground = rgb(0,0,224)
+  ?pen foreground = idx(4)
 PUSH "\e[91m"
-  ?pen foreground = rgb(255,64,64)
+  ?pen foreground = idx(9)
 PUSH "\e[38:2:10:20:30m"
   ?pen foreground = rgb(10,20,30)
 PUSH "\e[38:5:1m"
-  ?pen foreground = rgb(224,0,0)
+  ?pen foreground = idx(1)
 PUSH "\e[39m"
-  ?pen foreground = rgb(240,240,240)
+  ?pen foreground = rgb(240,240,240,is_default_fg)
 
 !Background
 PUSH "\e[41m"
-  ?pen background = rgb(224,0,0)
+  ?pen background = idx(1)
 PUSH "\e[42m"
-  ?pen background = rgb(0,224,0)
+  ?pen background = idx(2)
 PUSH "\e[44m"
-  ?pen background = rgb(0,0,224)
+  ?pen background = idx(4)
 PUSH "\e[101m"
-  ?pen background = rgb(255,64,64)
+  ?pen background = idx(9)
 PUSH "\e[48:2:10:20:30m"
   ?pen background = rgb(10,20,30)
 PUSH "\e[48:5:1m"
-  ?pen background = rgb(224,0,0)
+  ?pen background = idx(1)
 PUSH "\e[49m"
-  ?pen background = rgb(0,0,0)
+  ?pen background = rgb(0,0,0,is_default_bg)
 
 !Bold+ANSI colour == highbright
 PUSH "\e[m\e[1;37m"
   ?pen bold = on
-  ?pen foreground = rgb(255,255,255)
+  ?pen foreground = idx(15)
 PUSH "\e[m\e[37;1m"
   ?pen bold = on
-  ?pen foreground = rgb(255,255,255)
+  ?pen foreground = idx(15)

src/libvterm/t/31state_rep.test

@@ -0,0 +1,128 @@
+INIT
+UTF8 1
+WANTSTATE g
+
+!REP no argument
+RESET
+PUSH "a\e[b"
+  putglyph 0x61 1 0,0
+  putglyph 0x61 1 0,1
+
+!REP zero (zero should be interpreted as one)
+RESET
+PUSH "a\e[0b"
+  putglyph 0x61 1 0,0
+  putglyph 0x61 1 0,1
+
+!REP lowercase a times two
+RESET
+PUSH "a\e[2b"
+  putglyph 0x61 1 0,0
+  putglyph 0x61 1 0,1
+  putglyph 0x61 1 0,2
+
+!REP with UTF-8 1 char
+# U+00E9 = 0xC3 0xA9  name: LATIN SMALL LETTER E WITH ACUTE
+RESET
+PUSH "\xC3\xA9\e[b"
+  putglyph 0xe9 1 0,0
+  putglyph 0xe9 1 0,1
+
+!REP with UTF-8 wide char
+# U+00E9 = 0xC3 0xA9  name: LATIN SMALL LETTER E WITH ACUTE
+RESET
+PUSH "\xEF\xBC\x90\e[b"
+  putglyph 0xff10 2 0,0
+  putglyph 0xff10 2 0,2
+
+!REP with UTF-8 combining character
+RESET
+PUSH "e\xCC\x81\e[b"
+  putglyph 0x65,0x301 1 0,0
+  putglyph 0x65,0x301 1 0,1
+
+!REP till end of line
+RESET
+PUSH "a\e[1000bb"
+  putglyph 0x61 1 0,0
+  putglyph 0x61 1 0,1
+  putglyph 0x61 1 0,2
+  putglyph 0x61 1 0,3
+  putglyph 0x61 1 0,4
+  putglyph 0x61 1 0,5
+  putglyph 0x61 1 0,6
+  putglyph 0x61 1 0,7
+  putglyph 0x61 1 0,8
+  putglyph 0x61 1 0,9
+  putglyph 0x61 1 0,10
+  putglyph 0x61 1 0,11
+  putglyph 0x61 1 0,12
+  putglyph 0x61 1 0,13
+  putglyph 0x61 1 0,14
+  putglyph 0x61 1 0,15
+  putglyph 0x61 1 0,16
+  putglyph 0x61 1 0,17
+  putglyph 0x61 1 0,18
+  putglyph 0x61 1 0,19
+  putglyph 0x61 1 0,20
+  putglyph 0x61 1 0,21
+  putglyph 0x61 1 0,22
+  putglyph 0x61 1 0,23
+  putglyph 0x61 1 0,24
+  putglyph 0x61 1 0,25
+  putglyph 0x61 1 0,26
+  putglyph 0x61 1 0,27
+  putglyph 0x61 1 0,28
+  putglyph 0x61 1 0,29
+  putglyph 0x61 1 0,30
+  putglyph 0x61 1 0,31
+  putglyph 0x61 1 0,32
+  putglyph 0x61 1 0,33
+  putglyph 0x61 1 0,34
+  putglyph 0x61 1 0,35
+  putglyph 0x61 1 0,36
+  putglyph 0x61 1 0,37
+  putglyph 0x61 1 0,38
+  putglyph 0x61 1 0,39
+  putglyph 0x61 1 0,40
+  putglyph 0x61 1 0,41
+  putglyph 0x61 1 0,42
+  putglyph 0x61 1 0,43
+  putglyph 0x61 1 0,44
+  putglyph 0x61 1 0,45
+  putglyph 0x61 1 0,46
+  putglyph 0x61 1 0,47
+  putglyph 0x61 1 0,48
+  putglyph 0x61 1 0,49
+  putglyph 0x61 1 0,50
+  putglyph 0x61 1 0,51
+  putglyph 0x61 1 0,52
+  putglyph 0x61 1 0,53
+  putglyph 0x61 1 0,54
+  putglyph 0x61 1 0,55
+  putglyph 0x61 1 0,56
+  putglyph 0x61 1 0,57
+  putglyph 0x61 1 0,58
+  putglyph 0x61 1 0,59
+  putglyph 0x61 1 0,60
+  putglyph 0x61 1 0,61
+  putglyph 0x61 1 0,62
+  putglyph 0x61 1 0,63
+  putglyph 0x61 1 0,64
+  putglyph 0x61 1 0,65
+  putglyph 0x61 1 0,66
+  putglyph 0x61 1 0,67
+  putglyph 0x61 1 0,68
+  putglyph 0x61 1 0,69
+  putglyph 0x61 1 0,70
+  putglyph 0x61 1 0,71
+  putglyph 0x61 1 0,72
+  putglyph 0x61 1 0,73
+  putglyph 0x61 1 0,74
+  putglyph 0x61 1 0,75
+  putglyph 0x61 1 0,76
+  putglyph 0x61 1 0,77
+  putglyph 0x61 1 0,78
+  putglyph 0x61 1 0,79
+  putglyph 0x62 1 1,0
+

src/libvterm/t/32state_flow.test

@@ -0,0 +1,28 @@
+INIT
+WANTSTATE
+
+# Many of these test cases inspired by
+#   https://blueprints.launchpad.net/libvterm/+spec/reflow-cases
+
+!Spillover text marks continuation on second line
+RESET
+PUSH "A"x100
+PUSH "\r\n"
+  ?lineinfo 0 =
+  ?lineinfo 1 = cont
+
+!CRLF in column 80 does not mark continuation
+RESET
+PUSH "B"x80
+PUSH "\r\n"
+PUSH "B"x20
+PUSH "\r\n"
+  ?lineinfo 0 =
+  ?lineinfo 1 =
+
+!EL cancels continuation of following line
+RESET
+PUSH "D"x100
+  ?lineinfo 1 = cont
+PUSH "\eM\e[79G\e[K"
+  ?lineinfo 1 =

src/libvterm/t/60screen_ascii.test

@@ -1,12 +1,12 @@
 INIT
-WANTSCREEN c
+WANTSCREEN ac
 
 !Get
 RESET
 PUSH "ABC"
   movecursor 0,3
-  ?screen_chars 0,0,1,3 = 0x41,0x42,0x43
-  ?screen_chars 0,0,1,80 = 0x41,0x42,0x43
+  ?screen_chars 0,0,1,3 = "ABC"
+  ?screen_chars 0,0,1,80 = "ABC"
   ?screen_text 0,0,1,3 = 0x41,0x42,0x43
   ?screen_text 0,0,1,80 = 0x41,0x42,0x43
   ?screen_cell 0,0 = {0x41} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)
@@ -18,11 +18,11 @@ PUSH "ABC"
   ?screen_eol 0,3 = 1
 PUSH "\e[H"
   movecursor 0,0
-  ?screen_chars 0,0,1,80 = 0x41,0x42,0x43
+  ?screen_chars 0,0,1,80 = "ABC"
   ?screen_text 0,0,1,80 = 0x41,0x42,0x43
 PUSH "E"
   movecursor 0,1
-  ?screen_chars 0,0,1,80 = 0x45,0x42,0x43
+  ?screen_chars 0,0,1,80 = "EBC"
   ?screen_text 0,0,1,80 = 0x45,0x42,0x43
 
 WANTSCREEN -c
@@ -37,33 +37,33 @@ PUSH "ABCDE\e[H\e[K"
 RESET
 PUSH "ABC\e[H\e[@"
 PUSH "1"
-  ?screen_chars 0,0,1,80 = 0x31,0x41,0x42,0x43
+  ?screen_chars 0,0,1,80 = "1ABC"
 
 RESET
 PUSH "ABC\e[H\e[P"
-  ?screen_chars 0,0,1,1 = 0x42
-  ?screen_chars 0,1,1,2 = 0x43
-  ?screen_chars 0,0,1,80 = 0x42,0x43
+  ?screen_chars 0,0,1,1 = "B"
+  ?screen_chars 0,1,1,2 = "C"
+  ?screen_chars 0,0,1,80 = "BC"
 
 !Space padding
 RESET
 PUSH "Hello\e[CWorld"
-  ?screen_chars 0,0,1,80 = 0x48,0x65,0x6c,0x6c,0x6f,0x20,0x57,0x6f,0x72,0x6c,0x64
+  ?screen_chars 0,0,1,80 = "Hello World"
   ?screen_text 0,0,1,80 = 0x48,0x65,0x6c,0x6c,0x6f,0x20,0x57,0x6f,0x72,0x6c,0x64
 
 !Linefeed padding
 RESET
 PUSH "Hello\r\nWorld"
-  ?screen_chars 0,0,2,80 = 0x48,0x65,0x6c,0x6c,0x6f,0x0a,0x57,0x6f,0x72,0x6c,0x64
+  ?screen_chars 0,0,2,80 = "Hello\nWorld"
   ?screen_text 0,0,2,80 = 0x48,0x65,0x6c,0x6c,0x6f,0x0a,0x57,0x6f,0x72,0x6c,0x64
 
 !Altscreen
 RESET
 PUSH "P"
-  ?screen_chars 0,0,1,80 = 0x50
+  ?screen_chars 0,0,1,80 = "P"
 PUSH "\e[?1049h"
   ?screen_chars 0,0,1,80 = 
 PUSH "\e[2K\e[HA"
-  ?screen_chars 0,0,1,80 = 0x41
+  ?screen_chars 0,0,1,80 = "A"
 PUSH "\e[?1049l"
-  ?screen_chars 0,0,1,80 = 0x50
+  ?screen_chars 0,0,1,80 = "P"

src/libvterm/t/62screen_damage.test

@@ -1,5 +1,5 @@
 INIT
-WANTSCREEN Db
+WANTSCREEN aDb
 
 !Putglyph
 RESET
@@ -152,4 +152,4 @@ PUSH "\e[25H\r\nABCDE\b\b\b\e[2P\r\n"
 DAMAGEFLUSH
   moverect 1..25,0..80 -> 0..24,0..80
   damage 24..25,0..80
-  ?screen_chars 23,0,24,5 = 0x41,0x42,0x45
+  ?screen_chars 23,0,24,5 = "ABE"

src/libvterm/t/63screen_resize.test

@@ -6,42 +6,42 @@ WANTSCREEN
 RESET
 RESIZE 25,80
 PUSH "AB\r\nCD"
-  ?screen_chars 0,0,1,80 = 0x41,0x42
-  ?screen_chars 1,0,2,80 = 0x43,0x44
+  ?screen_chars 0,0,1,80 = "AB"
+  ?screen_chars 1,0,2,80 = "CD"
 RESIZE 25,100
-  ?screen_chars 0,0,1,100 = 0x41,0x42
-  ?screen_chars 1,0,2,100 = 0x43,0x44
+  ?screen_chars 0,0,1,100 = "AB"
+  ?screen_chars 1,0,2,100 = "CD"
 
 !Resize wider allows print in new area
 RESET
 RESIZE 25,80
 PUSH "AB\e[79GCD"
-  ?screen_chars 0,0,1,2 = 0x41,0x42
-  ?screen_chars 0,78,1,80 = 0x43,0x44
+  ?screen_chars 0,0,1,2 = "AB"
+  ?screen_chars 0,78,1,80 = "CD"
 RESIZE 25,100
-  ?screen_chars 0,0,1,2 = 0x41,0x42
-  ?screen_chars 0,78,1,80 = 0x43,0x44
+  ?screen_chars 0,0,1,2 = "AB"
+  ?screen_chars 0,78,1,80 = "CD"
 PUSH "E"
-  ?screen_chars 0,78,1,81 = 0x43,0x44,0x45
+  ?screen_chars 0,78,1,81 = "CDE"
 
 !Resize shorter with blanks just truncates
 RESET
 RESIZE 25,80
 PUSH "Top\e[10HLine 10"
-  ?screen_chars 0,0,1,80 = 0x54,0x6f,0x70
-  ?screen_chars 9,0,10,80 = 0x4c,0x69,0x6e,0x65,0x20,0x31,0x30
+  ?screen_chars 0,0,1,80 = "Top"
+  ?screen_chars 9,0,10,80 = "Line 10"
   ?cursor = 9,7
 RESIZE 20,80
-  ?screen_chars 0,0,1,80 = 0x54,0x6f,0x70
-  ?screen_chars 9,0,10,80 = 0x4c,0x69,0x6e,0x65,0x20,0x31,0x30
+  ?screen_chars 0,0,1,80 = "Top"
+  ?screen_chars 9,0,10,80 = "Line 10"
   ?cursor = 9,7
 
 !Resize shorter with content must scroll
 RESET
 RESIZE 25,80
 PUSH "Top\e[25HLine 25\e[15H"
-  ?screen_chars 0,0,1,80 = 0x54,0x6f,0x70
-  ?screen_chars 24,0,25,80 = 0x4c,0x69,0x6e,0x65,0x20,0x32,0x35
+  ?screen_chars 0,0,1,80 = "Top"
+  ?screen_chars 24,0,25,80 = "Line 25"
   ?cursor = 14,0
 WANTSCREEN b
 RESIZE 20,80
@@ -51,7 +51,7 @@ RESIZE 20,80
   sb_pushline 80 =
   sb_pushline 80 =
   ?screen_chars 0,0,1,80 = 
-  ?screen_chars 19,0,20,80 = 0x4c,0x69,0x6e,0x65,0x20,0x32,0x35
+  ?screen_chars 19,0,20,80 = "Line 25"
   ?cursor = 9,0
 
 !Resize shorter does not lose line with cursor
@@ -62,11 +62,11 @@ RESIZE 25,80
 WANTSCREEN b
 PUSH "\e[24HLine 24\r\nLine 25\r\n"
   sb_pushline 80 =
-  ?screen_chars 23,0,24,10 = 0x4c,0x69,0x6e,0x65,0x20,0x32,0x35
+  ?screen_chars 23,0,24,10 = "Line 25"
   ?cursor = 24,0
 RESIZE 24,80
   sb_pushline 80 =
-  ?screen_chars 22,0,23,10 = 0x4c,0x69,0x6e,0x65,0x20,0x32,0x35
+  ?screen_chars 22,0,23,10 = "Line 25"
   ?cursor = 23,0
 
 !Resize taller attempts to pop scrollback
@@ -74,8 +74,8 @@ RESET
 WANTSCREEN -b
 RESIZE 25,80
 PUSH "Line 1\e[25HBottom\e[15H"
-  ?screen_chars 0,0,1,80 = 0x4c,0x69,0x6e,0x65,0x20,0x31
-  ?screen_chars 24,0,25,80 = 0x42,0x6f,0x74,0x74,0x6f,0x6d
+  ?screen_chars 0,0,1,80 = "Line 1"
+  ?screen_chars 24,0,25,80 = "Bottom"
   ?cursor = 14,0
 WANTSCREEN b
 RESIZE 30,80
@@ -84,7 +84,18 @@ RESIZE 30,80
   sb_popline 80
   sb_popline 80
   sb_popline 80
-  ?screen_chars 0,0,1,80 = 0x41,0x42,0x43,0x44,0x45
-  ?screen_chars 5,0,6,80 = 0x4c,0x69,0x6e,0x65,0x20,0x31
-  ?screen_chars 29,0,30,80 = 0x42,0x6f,0x74,0x74,0x6f,0x6d
+  ?screen_chars 0,0,1,80 = "ABCDE"
+  ?screen_chars 5,0,6,80 = "Line 1"
+  ?screen_chars 29,0,30,80 = "Bottom"
   ?cursor = 19,0
+WANTSCREEN -b
+
+!Resize can operate on altscreen
+RESET
+WANTSCREEN a
+RESIZE 25,80
+PUSH "Main screen\e[?1049h\e[HAlt screen"
+RESIZE 30,80
+  ?screen_chars 0,0,1,3 = "Alt"
+PUSH "\e[?1049l"
+  ?screen_chars 0,0,1,3 = "Mai"

src/libvterm/t/64screen_pen.test

@@ -29,27 +29,27 @@ PUSH "\e[11mF\e[m"
 
 !Foreground
 PUSH "\e[31mG\e[m"
-  ?screen_cell 0,6 = {0x47} width=1 attrs={} fg=rgb(224,0,0) bg=rgb(0,0,0)
+  ?screen_cell 0,6 = {0x47} width=1 attrs={} fg=idx(1) bg=rgb(0,0,0)
 
 !Background
 PUSH "\e[42mH\e[m"
-  ?screen_cell 0,7 = {0x48} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,224,0)
+  ?screen_cell 0,7 = {0x48} width=1 attrs={} fg=rgb(240,240,240) bg=idx(2)
 
 !EL sets reverse and colours to end of line
 PUSH "\e[H\e[7;33;44m\e[K"
-  ?screen_cell 0,0  = {} width=1 attrs={R} fg=rgb(224,224,0) bg=rgb(0,0,224)
-  ?screen_cell 0,79 = {} width=1 attrs={R} fg=rgb(224,224,0) bg=rgb(0,0,224)
+  ?screen_cell 0,0  = {} width=1 attrs={R} fg=idx(3) bg=idx(4)
+  ?screen_cell 0,79 = {} width=1 attrs={R} fg=idx(3) bg=idx(4)
 
 !DECSCNM xors reverse for entire screen
 PUSH "\e[?5h"
-  ?screen_cell 0,0  = {} width=1 attrs={} fg=rgb(224,224,0) bg=rgb(0,0,224)
-  ?screen_cell 0,79 = {} width=1 attrs={} fg=rgb(224,224,0) bg=rgb(0,0,224)
+  ?screen_cell 0,0  = {} width=1 attrs={} fg=idx(3) bg=idx(4)
+  ?screen_cell 0,79 = {} width=1 attrs={} fg=idx(3) bg=idx(4)
   ?screen_cell 1,0  = {} width=1 attrs={R} fg=rgb(240,240,240) bg=rgb(0,0,0)
 PUSH "\e[?5\$p"
   output "\e[?5;1\$y"
 PUSH "\e[?5l"
-  ?screen_cell 0,0  = {} width=1 attrs={R} fg=rgb(224,224,0) bg=rgb(0,0,224)
-  ?screen_cell 0,79 = {} width=1 attrs={R} fg=rgb(224,224,0) bg=rgb(0,0,224)
+  ?screen_cell 0,0  = {} width=1 attrs={R} fg=idx(3) bg=idx(4)
+  ?screen_cell 0,79 = {} width=1 attrs={R} fg=idx(3) bg=idx(4)
   ?screen_cell 1,0  = {} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)
 PUSH "\e[?5\$p"
   output "\e[?5;2\$y"

src/libvterm/t/67screen_dbl_wh.test

@@ -30,3 +30,9 @@ PUSH "abcde"
   ?screen_cell 0,0 = {0x61} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)
 PUSH "\e#6"
   ?screen_cell 0,0 = {0x61} width=1 attrs={} dwl fg=rgb(240,240,240) bg=rgb(0,0,0)
+
+!DWL doesn't spill over on scroll
+RESET
+PUSH "\e[25H\e#6Final\r\n"
+  ?screen_cell 23,0 = {0x46} width=1 attrs={} dwl fg=rgb(240,240,240) bg=rgb(0,0,0)
+  ?screen_cell 24,0 = {} width=1 attrs={} fg=rgb(240,240,240) bg=rgb(0,0,0)

src/libvterm/t/68screen_termprops.test

@@ -14,4 +14,4 @@ PUSH "\e[?25l"
 
 !Title
 PUSH "\e]2;Here is my title\a"
-  settermprop 4 "Here is my title"
+  settermprop 4 ["Here is my title"]

src/libvterm/t/harness.c

@@ -47,6 +47,7 @@ static VTermKey strp_key(char *str)
     { "Tab",   VTERM_KEY_TAB },
     { "Enter", VTERM_KEY_ENTER },
     { "KP0",   VTERM_KEY_KP_0 },
+    { "F1",    VTERM_KEY_FUNCTION(1) },
     { NULL,    VTERM_KEY_NONE },
   };
   int i;
@@ -59,6 +60,26 @@ static VTermKey strp_key(char *str)
   return VTERM_KEY_NONE;
 }
 
+static void print_color(const VTermColor *col)
+{
+  if (VTERM_COLOR_IS_RGB(col)) {
+    printf("rgb(%d,%d,%d", col->red, col->green, col->blue);
+  }
+  else if (VTERM_COLOR_IS_INDEXED(col)) {
+    printf("idx(%d", col->index);
+  }
+  else {
+    printf("invalid(%d", col->type);
+  }
+  if (VTERM_COLOR_IS_DEFAULT_FG(col)) {
+    printf(",is_default_fg");
+  }
+  if (VTERM_COLOR_IS_DEFAULT_BG(col)) {
+    printf(",is_default_bg");
+  }
+  printf(")");
+}
+
 static VTerm *vt;
 static VTermState *state;
 static VTermScreen *screen;
@@ -152,21 +173,44 @@ static int parser_csi(const char *leader, const long args[], int argcount, const
   return 1;
 }
 
-static int parser_osc(const char *command, size_t cmdlen, void *user UNUSED)
+static int parser_osc(int command, VTermStringFragment frag, void *user UNUSED)
 {
 
   printf("osc ");
-  printhex(command, cmdlen);
+
+  if(frag.initial) {
+    if(command == -1)
+      printf("[");
+    else
+      printf("[%d;", command);
+  }
+
+  printhex(frag.str, frag.len);
+
+  if(frag.final)
+    printf("]");
+
   printf("\n");
 
   return 1;
 }
 
-static int parser_dcs(const char *command, size_t cmdlen, void *user UNUSED)
+static int parser_dcs(const char *command, size_t commandlen, VTermStringFragment frag, void *user UNUSED)
 {
-
   printf("dcs ");
-  printhex(command, cmdlen);
+
+  if(frag.initial) {
+    size_t i;
+    printf("[");
+    for(i = 0; i < commandlen; i++)
+      printf("%02x", command[i]);
+  }
+
+  printhex(frag.str, frag.len);
+
+  if(frag.final)
+    printf("]");
+
   printf("\n");
 
   return 1;
@@ -182,7 +226,14 @@ static VTermParserCallbacks parser_cbs = {
   NULL // resize
 };
 
-// These callbacks are shared by State and Screen
+static VTermStateFallbacks fallbacks = {
+  parser_control, // control
+  parser_csi, // csi
+  parser_osc, // osc
+  parser_dcs // dcs
+};
+
+/* These callbacks are shared by State and Screen */
 
 static int want_movecursor = 0;
 static VTermPos state_pos;
@@ -238,10 +289,13 @@ static int settermprop(VTermProp prop, VTermValue *val, void *user UNUSED)
     printf("settermprop %d %d\n", prop, val->number);
     return 1;
   case VTERM_VALUETYPE_STRING:
-    printf("settermprop %d \"%s\"\n", prop, val->string);
+    printf("settermprop %d %s\"%.*s\"%s\n", prop,
+        val->string.initial ? "[" : "", val->string.len, val->string.str, val->string.final ? "]" : "");
     return 1;
   case VTERM_VALUETYPE_COLOR:
-    printf("settermprop %d rgb(%d,%d,%d)\n", prop, val->color.red, val->color.green, val->color.blue);
+    printf("settermprop %d ", prop);
+    print_color(&val->color);
+    printf("\n");
     return 1;
 
   case VTERM_N_VALUETYPES:
@@ -261,7 +315,7 @@ static int state_putglyph(VTermGlyphInfo *info, VTermPos pos, void *user UNUSED)
     return 1;
 
   printf("putglyph ");
-  for(i = 0; info->chars[i]; i++)
+  for(i = 0; i < VTERM_MAX_CHARS_PER_CELL && info->chars[i]; i++)
     printf(i ? ",%x" : "%x", info->chars[i]);
   printf(" %d %d,%d", info->width, pos.row, pos.col);
   if(info->protected_cell)
@@ -294,6 +348,7 @@ static struct {
   int italic;
   int blink;
   int reverse;
+  int conceal;
   int strike;
   int font;
   VTermColor foreground;
@@ -317,6 +372,9 @@ static int state_setpenattr(VTermAttr attr, VTermValue *val, void *user UNUSED)
   case VTERM_ATTR_REVERSE:
     state_pen.reverse = val->boolean;
     break;
+  case VTERM_ATTR_CONCEAL:
+    state_pen.conceal = val->boolean;
+    break;
   case VTERM_ATTR_STRIKE:
     state_pen.strike = val->boolean;
     break;
@@ -482,6 +540,9 @@ int main(int argc UNUSED, char **argv UNUSED)
     if(streq(line, "INIT")) {
       if(!vt)
         vt = vterm_new(25, 80);
+
+      // Somehow this makes tests fail
+      // vterm_output_set_callback(vt, term_output, NULL);
     }
 
     else if(streq(line, "WANTPARSER")) {
@@ -524,7 +585,7 @@ int main(int argc UNUSED, char **argv UNUSED)
           want_settermprop = sense;
           break;
         case 'f':
-          vterm_state_set_unrecognised_fallbacks(state, sense ? &parser_cbs : NULL, NULL);
+          vterm_state_set_unrecognised_fallbacks(state, sense ? &fallbacks : NULL, NULL);
           break;
         default:
           fprintf(stderr, "Unrecognised WANTSTATE flag '%c'\n", line[i]);
@@ -536,7 +597,6 @@ int main(int argc UNUSED, char **argv UNUSED)
       int sense = 1;
       if(!screen)
         screen = vterm_obtain_screen(vt);
-      vterm_screen_enable_altscreen(screen, 1);
       vterm_screen_set_callbacks(screen, &screen_cbs, NULL);
 
       while(line[i] == ' ')
@@ -546,6 +606,9 @@ int main(int argc UNUSED, char **argv UNUSED)
         case '-':
           sense = 0;
           break;
+        case 'a':
+          vterm_screen_enable_altscreen(screen, 1);
+          break;
         case 'd':
           want_screen_damage = sense;
           break;
@@ -793,14 +856,35 @@ int main(int argc UNUSED, char **argv UNUSED)
             printf("%d\n", state_pen.font);
         }
         else if(streq(linep, "foreground")) {
-          printf("rgb(%d,%d,%d)\n", state_pen.foreground.red, state_pen.foreground.green, state_pen.foreground.blue);
+          print_color(&state_pen.foreground);
+          printf("\n");
         }
         else if(streq(linep, "background")) {
-          printf("rgb(%d,%d,%d)\n", state_pen.background.red, state_pen.background.green, state_pen.background.blue);
+          print_color(&state_pen.background);
+          printf("\n");
         }
         else
           printf("?\n");
       }
+      else if(strstartswith(line, "?lineinfo ")) {
+        char *linep = line + 10;
+        int row;
+        const VTermLineInfo *info;
+        while(linep[0] == ' ')
+          linep++;
+        if(sscanf(linep, "%d", &row) < 1) {
+          printf("! lineinfo unrecognised input\n");
+          goto abort_line;
+        }
+        info = vterm_state_get_lineinfo(state, row);
+        if(info->doublewidth)
+          printf("dwl ");
+        if(info->doubleheight)
+          printf("dhl ");
+        if(info->continuation)
+          printf("cont ");
+        printf("\n");
+      }
       else if(strstartswith(line, "?screen_chars ")) {
         char *linep = line + 13;
         VTermRect rect;
@@ -892,8 +976,13 @@ int main(int argc UNUSED, char **argv UNUSED)
         printf("} ");
         if(cell.attrs.dwl)       printf("dwl ");
         if(cell.attrs.dhl)       printf("dhl-%s ", cell.attrs.dhl == 2 ? "bottom" : "top");
-        printf("fg=rgb(%d,%d,%d) ",  cell.fg.red, cell.fg.green, cell.fg.blue);
-        printf("bg=rgb(%d,%d,%d)\n", cell.bg.red, cell.bg.green, cell.bg.blue);
+        printf("fg=");
+        vterm_screen_convert_color_to_rgb(screen, &cell.fg);
+        print_color(&cell.fg);
+        printf(" bg=");
+        vterm_screen_convert_color_to_rgb(screen, &cell.bg);
+        print_color(&cell.bg);
+        printf("\n");
       }
       else if(strstartswith(line, "?screen_eol ")) {
         VTermPos pos;

src/libvterm/t/run-test.pl

@@ -8,14 +8,17 @@ use IPC::Open2 qw( open2 );
 use POSIX qw( WIFEXITED WEXITSTATUS WIFSIGNALED WTERMSIG );
 
 my $VALGRIND = 0;
+my $EXECUTABLE = "t/.libs/harness";
 GetOptions(
    'valgrind|v+' => \$VALGRIND,
+   'executable|e=s' => \$EXECUTABLE,
+   'fail-early|F' => \(my $FAIL_EARLY),
 ) or exit 1;
 
 my ( $hin, $hout, $hpid );
 {
    local $ENV{LD_LIBRARY_PATH} = ".libs";
-   my @command = "t/.libs/harness";
+   my @command = $EXECUTABLE;
    unshift @command, "valgrind", "--tool=memcheck", "--leak-check=yes", "--num-callers=25", "--log-file=valgrind.out", "--error-exitcode=126" if $VALGRIND;
 
    $hpid = open2 $hout, $hin, @command or die "Cannot open2 harness - $!";
@@ -26,6 +29,8 @@ my $exitcode = 0;
 my $command;
 my @expect;
 
+my $linenum = 0;
+
 sub do_onetest
 {
    $hin->print( "$command\n" );
@@ -39,7 +44,7 @@ sub do_onetest
       chomp $outline;
 
       if( !@expect ) {
-         print "# Test failed\n" unless $fail_printed++;
+         print "# line $linenum: Test failed\n" unless $fail_printed++;
          print "#    expected nothing more\n" .
                "#   Actual:   $outline\n";
          next;
@@ -49,18 +54,19 @@ sub do_onetest
 
       next if $expectation eq $outline;
 
-      print "# Test failed\n" unless $fail_printed++;
+      print "# line $linenum: Test failed\n" unless $fail_printed++;
       print "#   Expected: $expectation\n" .
             "#   Actual:   $outline\n";
    }
 
    if( @expect ) {
-      print "# Test failed\n" unless $fail_printed++;
+      print "# line $linenum: Test failed\n" unless $fail_printed++;
       print "#   Expected: $_\n" .
             "#    didn't happen\n" for @expect;
    }
 
    $exitcode = 1 if $fail_printed;
+   exit $exitcode if $exitcode and $FAIL_EARLY;
 }
 
 sub do_line
@@ -101,8 +107,15 @@ sub do_line
       elsif( $line =~ m/^csi (\S+) (.*)$/ ) {
          $line = sprintf "csi %02x %s", eval($1), $2; # TODO
       }
-      elsif( $line =~ m/^(escape|osc|dcs) (.*)$/ ) {
-         $line = "$1 " . join "", map sprintf("%02x", $_), unpack "C*", eval($2);
+      elsif( $line =~ m/^(osc) (\[\d+)? *(.*?)(\]?)$/ ) {
+         my ( $cmd, $initial, $data, $final ) = ( $1, $2, $3, $4 );
+         $initial //= "";
+         $initial .= ";" if $initial =~ m/\d+/;
+
+         $line = "$cmd $initial" . join( "", map sprintf("%02x", $_), unpack "C*", length $data ? eval($data) : "" ) . "$final";
+      }
+      elsif( $line =~ m/^(escape|dcs) (\[?)(.*?)(\]?)$/ ) {
+         $line = "$1 $2" . join( "", map sprintf("%02x", $_), unpack "C*", eval($3) ) . "$4";
       }
       elsif( $line =~ m/^putglyph (\S+) (.*)$/ ) {
          $line = "putglyph " . join( ",", map sprintf("%x", $_), eval($1) ) . " $2";
@@ -131,27 +144,35 @@ sub do_line
 
       $response = pack "C*", map hex, split m/,/, $response;
       if( $response ne $want ) {
-         print "# Assert ?screen_row $row failed:\n" .
+         print "# line $linenum: Assert ?screen_row $row failed:\n" .
                "# Expected: $want\n" .
                "# Actual:   $response\n";
          $exitcode = 1;
+         exit $exitcode if $exitcode and $FAIL_EARLY;
       }
    }
    # Assertions start with '?'
-   elsif( $line =~ s/^\?([a-z]+.*?=)\s+// ) {
+   elsif( $line =~ s/^\?([a-z]+.*?=)\s*// ) {
       do_onetest if defined $command;
 
       my ( $assertion ) = $1 =~ m/^(.*)\s+=/;
+      my $expectation = $line;
 
       $hin->print( "\?$assertion\n" );
       my $response = <$hout>; defined $response or wait, die "Test harness failed - $?\n";
-      chomp $response;
+      chomp $response; $response =~ s/^\s+|\s+$//g;
 
-      if( $response ne $line ) {
-         print "# Assert $assertion failed:\n" .
-               "# Expected: $line\n" .
+      # Some convenience formatting
+      if( $assertion =~ m/^screen_chars/ and $expectation =~ m/^"/ ) {
+         $expectation = join ",", map sprintf("0x%02x", ord $_), split m//, eval($expectation);
+      }
+
+      if( $response ne $expectation ) {
+         print "# line $linenum: Assert $assertion failed:\n" .
+               "# Expected: $expectation\n" .
                "# Actual:   $response\n";
          $exitcode = 1;
+         exit $exitcode if $exitcode and $FAIL_EARLY;
       }
    }
    # Test controls start with '$'
@@ -174,10 +195,13 @@ sub do_line
 open my $test, "<", $ARGV[0] or die "Cannot open test script $ARGV[0] - $!";
 
 while( my $line = <$test> ) {
+   $linenum++;
    $line =~ s/^\s+//;
-   next if $line =~ m/^(?:#|$)/;
-
    chomp $line;
+
+   next if $line =~ m/^(?:#|$)/;
+   last if $line eq "__END__";
+
    do_line( $line );
 }
 

src/os_amiga.c

@@ -977,7 +977,7 @@ mch_exit(int r)
  *	it sends a 0 to the console to make it back into a CON: from a RAW:
  */
     void
-mch_settmode(int tmode)
+mch_settmode(tmode_T tmode)
 {
 #if defined(__AROS__) || defined(__amigaos4__)
     if (!SetMode(raw_in, tmode == TMODE_RAW ? 1 : 0))
@@ -1387,7 +1387,11 @@ mch_call_shell(
     if ((mydir = CurrentDir(mydir)) != 0) // make sure we stay in the same directory
 	UnLock(mydir);
     if (tmode == TMODE_RAW)
+    {
+	// The shell may have messed with the mode, always set it.
+	cur_tmode = TMODE_UNKNOWN;
 	settmode(TMODE_RAW);		// set to raw mode
+    }
 #ifdef FEAT_TITLE
     resettitle();
 #endif

src/os_mswin.c

@@ -553,7 +553,7 @@ vim_stat(const char *name, stat_T *stp)
 
 #if (defined(FEAT_GUI_MSWIN) && !defined(VIMDLL)) || defined(PROTO)
     void
-mch_settmode(int tmode UNUSED)
+mch_settmode(tmode_T tmode UNUSED)
 {
     // nothing to do
 }

src/os_unix.c

@@ -215,7 +215,8 @@ static volatile sig_atomic_t in_mch_delay = FALSE; // sleeping in mch_delay()
 static int dont_check_job_ended = 0;
 #endif
 
-static int curr_tmode = TMODE_COOK;	// contains current terminal mode
+// Current terminal mode from mch_settmode().  Can differ from cur_tmode.
+static tmode_T mch_cur_tmode = TMODE_COOK;
 
 #ifdef USE_XSMP
 typedef struct
@@ -581,7 +582,7 @@ mch_total_mem(int special UNUSED)
     void
 mch_delay(long msec, int ignoreinput)
 {
-    int		old_tmode;
+    tmode_T	old_tmode;
 #ifdef FEAT_MZSCHEME
     long	total = msec; // remember original value
 #endif
@@ -591,9 +592,10 @@ mch_delay(long msec, int ignoreinput)
 	// Go to cooked mode without echo, to allow SIGINT interrupting us
 	// here.  But we don't want QUIT to kill us (CTRL-\ used in a
 	// shell may produce SIGQUIT).
+	// Only do this if sleeping for more than half a second.
 	in_mch_delay = TRUE;
-	old_tmode = curr_tmode;
-	if (curr_tmode == TMODE_RAW)
+	old_tmode = mch_cur_tmode;
+	if (mch_cur_tmode == TMODE_RAW && msec > 500)
 	    settmode(TMODE_SLEEP);
 
 	/*
@@ -650,7 +652,8 @@ mch_delay(long msec, int ignoreinput)
 	while (total > 0);
 #endif
 
-	settmode(old_tmode);
+	if (msec > 500)
+	    settmode(old_tmode);
 	in_mch_delay = FALSE;
     }
     else
@@ -3461,7 +3464,7 @@ mch_tcgetattr(int fd, void *term)
 }
 
     void
-mch_settmode(int tmode)
+mch_settmode(tmode_T tmode)
 {
     static int first = TRUE;
 
@@ -3558,7 +3561,7 @@ mch_settmode(int tmode)
 	ttybnew.sg_flags &= ~(ECHO);
     ioctl(read_cmd_fd, TIOCSETN, &ttybnew);
 #endif
-    curr_tmode = tmode;
+    mch_cur_tmode = tmode;
 }
 
 /*
@@ -4455,7 +4458,7 @@ mch_call_shell_system(
     char	*ifn = NULL;
     char	*ofn = NULL;
 #endif
-    int		tmode = cur_tmode;
+    tmode_T	tmode = cur_tmode;
     char_u	*newcmd;	// only needed for unix
     int		x;
 
@@ -4521,7 +4524,11 @@ mch_call_shell_system(
     }
 
     if (tmode == TMODE_RAW)
+    {
+	// The shell may have messed with the mode, always set it.
+	cur_tmode = TMODE_UNKNOWN;
 	settmode(TMODE_RAW);	// set to raw mode
+    }
 # ifdef FEAT_TITLE
     resettitle();
 # endif
@@ -4545,7 +4552,7 @@ mch_call_shell_fork(
     char_u	*cmd,
     int		options)	// SHELL_*, see vim.h
 {
-    int		tmode = cur_tmode;
+    tmode_T	tmode = cur_tmode;
     pid_t	pid;
     pid_t	wpid = 0;
     pid_t	wait_pid = 0;
@@ -4571,6 +4578,9 @@ mch_call_shell_fork(
     out_flush();
     if (options & SHELL_COOKED)
 	settmode(TMODE_COOK);		// set to normal mode
+    if (tmode == TMODE_RAW)
+	// The shell may have messed with the mode, always set it later.
+	cur_tmode = TMODE_UNKNOWN;
 
     if (unix_build_argv(cmd, &argv, &tofree1, &tofree2) == FAIL)
 	goto error;
@@ -5932,7 +5942,7 @@ mch_create_pty_channel(job_T *job, jobopt_T *options)
     void
 mch_breakcheck(int force)
 {
-    if ((curr_tmode == TMODE_RAW || force)
+    if ((mch_cur_tmode == TMODE_RAW || force)
 			       && RealWaitForChar(read_cmd_fd, 0L, NULL, NULL))
 	fill_input_buf(FALSE);
 }

src/os_vms.c

@@ -112,7 +112,7 @@ vul_item(ITEM *itm, short len, short cod, char *adr, int *ret)
 }
 
     void
-mch_settmode(int tmode)
+mch_settmode(tmode_T tmode)
 {
     int	status;
 

src/os_win32.c

@@ -822,7 +822,7 @@ win32_enable_privilege(LPTSTR lpszPrivilege, BOOL bEnable)
 #endif
 
 /*
- * Set "win8_or_later" and fill in "windowsVersion".
+ * Set "win8_or_later" and fill in "windowsVersion" if possible.
  */
     void
 PlatformId(void)
@@ -836,9 +836,10 @@ PlatformId(void)
 	ovi.dwOSVersionInfoSize = sizeof(ovi);
 	GetVersionEx(&ovi);
 
+#ifdef FEAT_EVAL
 	vim_snprintf(windowsVersion, sizeof(windowsVersion), "%d.%d",
 		(int)ovi.dwMajorVersion, (int)ovi.dwMinorVersion);
-
+#endif
 	if ((ovi.dwMajorVersion == 6 && ovi.dwMinorVersion >= 2)
 		|| ovi.dwMajorVersion > 6)
 	    win8_or_later = TRUE;
@@ -3600,7 +3601,7 @@ handler_routine(
  * set the tty in (raw) ? "raw" : "cooked" mode
  */
     void
-mch_settmode(int tmode)
+mch_settmode(tmode_T tmode)
 {
     DWORD cmodein;
     DWORD cmodeout;
@@ -4899,7 +4900,11 @@ mch_call_shell(
     }
 
     if (tmode == TMODE_RAW)
+    {
+	// The shell may have messed with the mode, always set it.
+	cur_tmode = TMODE_UNKNOWN;
 	settmode(TMODE_RAW);	// set to raw mode
+    }
 
     // Print the return value, unless "vimrun" was used.
     if (x != 0 && !(options & SHELL_SILENT) && !emsg_silent

src/popupwin.c

@@ -1757,6 +1757,27 @@ add_border_left_right_padding(win_T *wp)
     }
 }
 
+#ifdef FEAT_TERMINAL
+/*
+ * Return TRUE if there is any popup window with a terminal buffer.
+ */
+    static int
+popup_terminal_exists(void)
+{
+    win_T	*wp;
+    tabpage_T	*tp;
+
+    FOR_ALL_POPUPWINS(wp)
+	if (wp->w_buffer->b_term != NULL)
+	    return TRUE;
+    FOR_ALL_TABPAGES(tp)
+	FOR_ALL_POPUPWINS_IN_TAB(tp, wp)
+	    if (wp->w_buffer->b_term != NULL)
+		return TRUE;
+    return FALSE;
+}
+#endif
+
 /*
  * popup_create({text}, {options})
  * popup_atcursor({text}, {options})
@@ -1786,6 +1807,13 @@ popup_create(typval_T *argvars, typval_T *rettv, create_type_T type)
 		semsg(_(e_nobufnr), argvars[0].vval.v_number);
 		return NULL;
 	    }
+#ifdef FEAT_TERMINAL
+	    if (buf->b_term != NULL && popup_terminal_exists())
+	    {
+		emsg(_("E861: Cannot open a second popup with a terminal"));
+		return NULL;
+	    }
+#endif
 	}
 	else if (!(argvars[0].v_type == VAR_STRING
 			&& argvars[0].vval.v_string != NULL)

src/proto/os_amiga.pro

@@ -31,7 +31,7 @@ int mch_can_exe(char_u *name, char_u **path, int use_path);
 int mch_nodetype(char_u *name);
 void mch_early_init(void);
 void mch_exit(int r);
-void mch_settmode(int tmode);
+void mch_settmode(tmode_T tmode);
 int mch_get_shellsize(void);
 void mch_set_shellsize(void);
 void mch_new_shellsize(void);

src/proto/os_mswin.pro

@@ -10,7 +10,7 @@ int mch_FullName(char_u *fname, char_u *buf, int len, int force);
 int mch_isFullName(char_u *fname);
 void slash_adjust(char_u *p);
 int vim_stat(const char *name, stat_T *stp);
-void mch_settmode(int tmode);
+void mch_settmode(tmode_T tmode);
 int mch_get_shellsize(void);
 void mch_set_shellsize(void);
 void mch_new_shellsize(void);

src/proto/os_unix.pro

@@ -48,7 +48,7 @@ int mch_nodetype(char_u *name);
 void mch_early_init(void);
 void mch_free_mem(void);
 void mch_exit(int r);
-void mch_settmode(int tmode);
+void mch_settmode(tmode_T tmode);
 void get_stty(void);
 int get_tty_info(int fd, ttyinfo_T *info);
 void mch_setmouse(int on);

src/proto/os_vms.pro

@@ -1,5 +1,5 @@
 /* os_vms.c */
-void mch_settmode(int tmode);
+void mch_settmode(tmode_T tmode);
 int mch_get_shellsize(void);
 void mch_set_shellsize(void);
 char_u *mch_getenv(char_u *lognam);

src/proto/os_win32.pro

@@ -39,7 +39,7 @@ int mch_nodetype(char_u *name);
 vim_acl_T mch_get_acl(char_u *fname);
 void mch_set_acl(char_u *fname, vim_acl_T acl);
 void mch_free_acl(vim_acl_T acl);
-void mch_settmode(int tmode);
+void mch_settmode(tmode_T tmode);
 int mch_get_shellsize(void);
 void mch_set_shellsize(void);
 void mch_new_shellsize(void);

src/proto/term.pro

@@ -41,7 +41,7 @@ void win_new_shellsize(void);
 void shell_resized(void);
 void shell_resized_check(void);
 void set_shellsize(int width, int height, int mustset);
-void settmode(int tmode);
+void settmode(tmode_T tmode);
 void starttermcap(void);
 void stoptermcap(void);
 void may_req_termresponse(void);

src/term.c

@@ -2862,7 +2862,7 @@ term_color(char_u *s, int n)
 #if defined(FEAT_VTP) && defined(FEAT_TERMGUICOLORS)
 		  || (s[0] == ESC && s[1] == '|')
 #endif
-	          || (s[0] == CSI && (i = 1) == 1))
+		  || (s[0] == CSI && (i = 1) == 1))
 	      && s[i] != NUL
 	      && (STRCMP(s + i + 1, "%p1%dm") == 0
 		  || STRCMP(s + i + 1, "%dm") == 0)
@@ -3436,7 +3436,7 @@ set_shellsize(int width, int height, int mustset)
  * commands and Ex mode).
  */
     void
-settmode(int tmode)
+settmode(tmode_T tmode)
 {
 #ifdef FEAT_GUI
     // don't set the term where gvim was started to any mode
@@ -3447,14 +3447,14 @@ settmode(int tmode)
     if (full_screen)
     {
 	/*
-	 * When returning after calling a shell we want to really set the
-	 * terminal to raw mode, even though we think it already is, because
-	 * the shell program may have reset the terminal mode.
+	 * When returning after calling a shell cur_tmode is TMODE_UNKNOWN,
+	 * set the terminal to raw mode, even though we think it already is,
+	 * because the shell program may have reset the terminal mode.
 	 * When we think the terminal is normal, don't try to set it to
 	 * normal again, because that causes problems (logout!) on some
 	 * machines.
 	 */
-	if (tmode != TMODE_COOK || cur_tmode != TMODE_COOK)
+	if (tmode != cur_tmode)
 	{
 #ifdef FEAT_TERMRESPONSE
 # ifdef FEAT_GUI
@@ -3471,7 +3471,12 @@ settmode(int tmode)
 #endif
 	    if (tmode != TMODE_RAW)
 		mch_setmouse(FALSE);	// switch mouse off
-	    if (termcap_active)
+
+	    // Disable bracketed paste and modifyOtherKeys in cooked mode.
+	    // Avoid doing this too often, on some terminals the codes are not
+	    // handled properly.
+	    if (termcap_active && tmode != TMODE_SLEEP
+						   && cur_tmode != TMODE_SLEEP)
 	    {
 		if (tmode != TMODE_RAW)
 		{
@@ -6352,12 +6357,6 @@ static int grey_ramp[] = {
     0x80, 0x8A, 0x94, 0x9E, 0xA8, 0xB2, 0xBC, 0xC6, 0xD0, 0xDA, 0xE4, 0xEE
 };
 
-# ifdef FEAT_TERMINAL
-#  include "libvterm/include/vterm.h"  // for VTERM_ANSI_INDEX_NONE
-# else
-#  define VTERM_ANSI_INDEX_NONE 0
-# endif
-
 static char_u ansi_table[16][4] = {
 //   R    G    B   idx
   {  0,   0,   0,  1}, // black
@@ -6379,6 +6378,8 @@ static char_u ansi_table[16][4] = {
   {255, 255, 255, 16}, // white
 };
 
+#define ANSI_INDEX_NONE 0
+
     void
 cterm_color2rgb(int nr, char_u *r, char_u *g, char_u *b, char_u *ansi_idx)
 {
@@ -6398,7 +6399,7 @@ cterm_color2rgb(int nr, char_u *r, char_u *g, char_u *b, char_u *ansi_idx)
 	*r = cube_value[idx / 36 % 6];
 	*g = cube_value[idx / 6  % 6];
 	*b = cube_value[idx      % 6];
-	*ansi_idx = VTERM_ANSI_INDEX_NONE;
+	*ansi_idx = ANSI_INDEX_NONE;
     }
     else if (nr < 256)
     {
@@ -6407,14 +6408,14 @@ cterm_color2rgb(int nr, char_u *r, char_u *g, char_u *b, char_u *ansi_idx)
 	*r = grey_ramp[idx];
 	*g = grey_ramp[idx];
 	*b = grey_ramp[idx];
-	*ansi_idx = VTERM_ANSI_INDEX_NONE;
+	*ansi_idx = ANSI_INDEX_NONE;
     }
     else
     {
 	*r = 0;
 	*g = 0;
 	*b = 0;
-	*ansi_idx = 0;
+	*ansi_idx = ANSI_INDEX_NONE;
     }
 }
 #endif

src/term.h

@@ -209,6 +209,9 @@ extern char_u *(term_strings[]);    // current terminal strings
 #define T_SSI	(TERM_STR(KS_SSI))	// save icon text
 #define T_SRI	(TERM_STR(KS_SRI))	// restore icon text
 
-#define TMODE_COOK  0	// terminal mode for external cmds and Ex mode
-#define TMODE_SLEEP 1	// terminal mode for sleeping (cooked but no echo)
-#define TMODE_RAW   2	// terminal mode for Normal and Insert mode
+typedef enum {
+    TMODE_COOK,	    // terminal mode for external cmds and Ex mode
+    TMODE_SLEEP,    // terminal mode for sleeping (cooked but no echo)
+    TMODE_RAW,	    // terminal mode for Normal and Insert mode
+    TMODE_UNKNOWN   // after executing a shell
+} tmode_T;

src/terminal.c

@@ -162,6 +162,7 @@ struct terminal_S {
     char_u	*tl_cursor_color; // NULL or allocated
 
     int		tl_using_altscreen;
+    garray_T	tl_osc_buf;	    // incomplete OSC string
 };
 
 #define TMODE_ONCE 1	    // CTRL-\ CTRL-N used
@@ -445,6 +446,7 @@ term_start(
 #endif
     ga_init2(&term->tl_scrollback, sizeof(sb_line_T), 300);
     ga_init2(&term->tl_scrollback_postponed, sizeof(sb_line_T), 300);
+    ga_init2(&term->tl_osc_buf, sizeof(char), 300);
 
     CLEAR_FIELD(split_ea);
     if (opt->jo_curwin)
@@ -1015,6 +1017,7 @@ free_unused_terminals()
 	terminals_to_free = term->tl_next;
 
 	free_scrollback(term);
+	ga_clear(&term->tl_osc_buf);
 
 	term_free_vterm(term);
 	vim_free(term->tl_api);
@@ -1623,7 +1626,8 @@ cell2cellattr(const VTermScreenCell *cell, cellattr_T *attr)
     static int
 equal_celattr(cellattr_T *a, cellattr_T *b)
 {
-    // Comparing the colors should be sufficient.
+    // We only compare the RGB colors, ignoring the ANSI index and type.
+    // Thus black set explicitly is equal the background black.
     return a->fg.red == b->fg.red
 	&& a->fg.green == b->fg.green
 	&& a->fg.blue == b->fg.blue
@@ -2689,10 +2693,13 @@ color2index(VTermColor *color, int fg, int *boldp)
     int blue = color->blue;
     int green = color->green;
 
-    if (color->ansi_index != VTERM_ANSI_INDEX_NONE)
+    if (VTERM_COLOR_IS_DEFAULT_FG(color)
+	    || VTERM_COLOR_IS_DEFAULT_BG(color))
+	return 0;
+    if (VTERM_COLOR_IS_INDEXED(color))
     {
 	// The first 16 colors and default: use the ANSI index.
-	switch (color->ansi_index)
+	switch (color->index + 1)
 	{
 	    case  0: return 0;
 	    case  1: return lookup_color( 0, fg, boldp) + 1; // black
@@ -2998,22 +3005,27 @@ handle_settermprop(
 	void *user)
 {
     term_T	*term = (term_T *)user;
+    char_u	*strval = NULL;
 
     switch (prop)
     {
 	case VTERM_PROP_TITLE:
+	    strval = vim_strnsave((char_u *)value->string.str,
+						       (int)value->string.len);
+	    if (strval == NULL)
+		break;
 	    vim_free(term->tl_title);
 	    // a blank title isn't useful, make it empty, so that "running" is
 	    // displayed
-	    if (*skipwhite((char_u *)value->string) == NUL)
+	    if (*skipwhite(strval) == NUL)
 		term->tl_title = NULL;
 	    // Same as blank
 	    else if (term->tl_arg0_cmd != NULL
-		    && STRNCMP(term->tl_arg0_cmd, (char_u *)value->string,
+		    && STRNCMP(term->tl_arg0_cmd, strval,
 					  (int)STRLEN(term->tl_arg0_cmd)) == 0)
 		term->tl_title = NULL;
 	    // Empty corrupted data of winpty
-	    else if (STRNCMP("  - ", (char_u *)value->string, 4) == 0)
+	    else if (STRNCMP("  - ", strval, 4) == 0)
 		term->tl_title = NULL;
 #ifdef MSWIN
 	    else if (!enc_utf8 && enc_codepage > 0)
@@ -3022,8 +3034,8 @@ handle_settermprop(
 		int	length = 0;
 
 		MultiByteToWideChar_alloc(CP_UTF8, 0,
-			(char*)value->string, (int)STRLEN(value->string),
-								&ret, &length);
+			(char*)value->string.str,
+					(int)value->string.len, &ret, &length);
 		if (ret != NULL)
 		{
 		    WideCharToMultiByte_alloc(enc_codepage, 0,
@@ -3034,7 +3046,10 @@ handle_settermprop(
 	    }
 #endif
 	    else
-		term->tl_title = vim_strsave((char_u *)value->string);
+	    {
+		term->tl_title = vim_strsave(strval);
+		strval = NULL;
+	    }
 	    VIM_CLEAR(term->tl_status_text);
 	    if (term == curbuf->b_term)
 		maketitle();
@@ -3057,7 +3072,11 @@ handle_settermprop(
 	    break;
 
 	case VTERM_PROP_CURSORCOLOR:
-	    cursor_color_copy(&term->tl_cursor_color, (char_u*)value->string);
+	    strval = vim_strnsave((char_u *)value->string.str,
+						       (int)value->string.len);
+	    if (strval == NULL)
+		break;
+	    cursor_color_copy(&term->tl_cursor_color, strval);
 	    may_set_cursor_props(term);
 	    break;
 
@@ -3069,6 +3088,8 @@ handle_settermprop(
 	default:
 	    break;
     }
+    vim_free(strval);
+
     // Always return 1, otherwise vterm doesn't store the value internally.
     return 1;
 }
@@ -3815,7 +3836,14 @@ term_get_attr(win_T *wp, linenr_T lnum, int col)
     static void
 cterm_color2vterm(int nr, VTermColor *rgb)
 {
-    cterm_color2rgb(nr, &rgb->red, &rgb->green, &rgb->blue, &rgb->ansi_index);
+    cterm_color2rgb(nr, &rgb->red, &rgb->green, &rgb->blue, &rgb->index);
+    if (rgb->index == 0)
+	rgb->type = VTERM_COLOR_RGB;
+    else
+    {
+	rgb->type = VTERM_COLOR_INDEXED;
+	--rgb->index;
+    }
 }
 
 /*
@@ -3847,7 +3875,8 @@ init_default_colors(term_T *term, win_T *wp)
     }
     fg->red = fg->green = fg->blue = fgval;
     bg->red = bg->green = bg->blue = bgval;
-    fg->ansi_index = bg->ansi_index = VTERM_ANSI_INDEX_DEFAULT;
+    fg->type = VTERM_COLOR_RGB | VTERM_COLOR_DEFAULT_FG;
+    bg->type = VTERM_COLOR_RGB | VTERM_COLOR_DEFAULT_BG;
 
     // The 'wincolor' or the highlight group overrules the defaults.
     if (wp != NULL && *wp->w_p_wcr != NUL)
@@ -4181,21 +4210,32 @@ handle_call_command(term_T *term, channel_T *channel, listitem_T *item)
  * We recognize a terminal API command.
  */
     static int
-parse_osc(const char *command, size_t cmdlen, void *user)
+parse_osc(int command, VTermStringFragment frag, void *user)
 {
     term_T	*term = (term_T *)user;
     js_read_T	reader;
     typval_T	tv;
     channel_T	*channel = term->tl_job == NULL ? NULL
 						    : term->tl_job->jv_channel;
+    garray_T	*gap = &term->tl_osc_buf;
 
     // We recognize only OSC 5 1 ; {command}
-    if (cmdlen < 3 || STRNCMP(command, "51;", 3) != 0)
-	return 0; // not handled
+    if (command != 51)
+	return 0;
 
-    reader.js_buf = vim_strnsave((char_u *)command + 3, (int)(cmdlen - 3));
-    if (reader.js_buf == NULL)
+    // Concatenate what was received until the final piece is found.
+    if (ga_grow(gap, (int)frag.len + 1) == FAIL)
+    {
+	ga_clear(gap);
 	return 1;
+    }
+    mch_memmove((char *)gap->ga_data + gap->ga_len, frag.str, frag.len);
+    gap->ga_len += frag.len;
+    if (!frag.final)
+	return 1;
+
+    ((char *)gap->ga_data)[gap->ga_len] = 0;
+    reader.js_buf = gap->ga_data;
     reader.js_fill = NULL;
     reader.js_used = 0;
     if (json_decode(&reader, &tv, 0) == OK
@@ -4229,7 +4269,7 @@ parse_osc(const char *command, size_t cmdlen, void *user)
     else
 	ch_log(channel, "Invalid JSON received");
 
-    vim_free(reader.js_buf);
+    ga_clear(gap);
     clear_tv(&tv);
     return 1;
 }
@@ -4293,14 +4333,11 @@ parse_csi(
     return 1;
 }
 
-static VTermParserCallbacks parser_fallbacks = {
-  NULL,		// text
+static VTermStateFallbacks state_fallbacks = {
   NULL,		// control
-  NULL,		// escape
   parse_csi,	// csi
   parse_osc,	// osc
-  NULL,		// dcs
-  NULL		// resize
+  NULL		// dcs
 };
 
 /*
@@ -4383,7 +4420,7 @@ create_vterm(term_T *term, int rows, int cols)
     value.boolean = 0;
 #endif
     vterm_state_set_termprop(state, VTERM_PROP_CURSORBLINK, &value);
-    vterm_state_set_unrecognised_fallbacks(state, &parser_fallbacks, term);
+    vterm_state_set_unrecognised_fallbacks(state, &state_fallbacks, term);
 
     return OK;
 }
@@ -4484,21 +4521,30 @@ term_get_buf(typval_T *argvars, char *where)
     return buf;
 }
 
-    static int
-same_color(VTermColor *a, VTermColor *b)
+    static void
+clear_cell(VTermScreenCell *cell)
 {
-    return a->red == b->red
-	&& a->green == b->green
-	&& a->blue == b->blue
-	&& a->ansi_index == b->ansi_index;
+    CLEAR_FIELD(*cell);
+    cell->fg.type = VTERM_COLOR_DEFAULT_FG;
+    cell->bg.type = VTERM_COLOR_DEFAULT_BG;
 }
 
     static void
 dump_term_color(FILE *fd, VTermColor *color)
 {
+    int index;
+
+    if (VTERM_COLOR_IS_INDEXED(color))
+	index = color->index + 1;
+    else if (color->type == 0)
+	// use RGB values
+	index = 255;
+    else
+	// default color
+	index = 0;
     fprintf(fd, "%02x%02x%02x%d",
 	    (int)color->red, (int)color->green, (int)color->blue,
-	    (int)color->ansi_index);
+	    index);
 }
 
 /*
@@ -4582,7 +4628,7 @@ f_term_dumpwrite(typval_T *argvars, typval_T *rettv UNUSED)
 	return;
     }
 
-    CLEAR_FIELD(prev_cell);
+    clear_cell(&prev_cell);
 
     screen = vterm_obtain_screen(term->tl_vterm);
     state = vterm_obtain_state(term->tl_vterm);
@@ -4604,7 +4650,7 @@ f_term_dumpwrite(typval_T *argvars, typval_T *rettv UNUSED)
 						 && pos.row == cursor_pos.row);
 
 	    if (vterm_screen_get_cell(screen, pos, &cell) == 0)
-		CLEAR_FIELD(cell);
+		clear_cell(&cell);
 
 	    for (i = 0; i < VTERM_MAX_CHARS_PER_CELL; ++i)
 	    {
@@ -4624,8 +4670,8 @@ f_term_dumpwrite(typval_T *argvars, typval_T *rettv UNUSED)
 	    }
 	    same_attr = vtermAttr2hl(cell.attrs)
 					       == vtermAttr2hl(prev_cell.attrs)
-			&& same_color(&cell.fg, &prev_cell.fg)
-			&& same_color(&cell.bg, &prev_cell.bg);
+			&& vterm_color_is_equal(&cell.fg, &prev_cell.fg)
+			&& vterm_color_is_equal(&cell.bg, &prev_cell.bg);
 	    if (same_chars && cell.width == prev_cell.width && same_attr
 							     && !is_cursor_pos)
 	    {
@@ -4672,14 +4718,14 @@ f_term_dumpwrite(typval_T *argvars, typval_T *rettv UNUSED)
 		    else
 		    {
 			fprintf(fd, "%d", vtermAttr2hl(cell.attrs));
-			if (same_color(&cell.fg, &prev_cell.fg))
+			if (vterm_color_is_equal(&cell.fg, &prev_cell.fg))
 			    fputs("&", fd);
 			else
 			{
 			    fputs("#", fd);
 			    dump_term_color(fd, &cell.fg);
 			}
-			if (same_color(&cell.bg, &prev_cell.bg))
+			if (vterm_color_is_equal(&cell.bg, &prev_cell.bg))
 			    fputs("&", fd);
 			else
 			{
@@ -4722,6 +4768,14 @@ append_cell(garray_T *gap, cellattr_T *cell)
     }
 }
 
+    static void
+clear_cellattr(cellattr_T *cell)
+{
+    CLEAR_FIELD(*cell);
+    cell->fg.type = VTERM_COLOR_DEFAULT_FG;
+    cell->bg.type = VTERM_COLOR_DEFAULT_BG;
+}
+
 /*
  * Read the dump file from "fd" and append lines to the current buffer.
  * Return the cell width of the longest line.
@@ -4742,8 +4796,8 @@ read_dump_file(FILE *fd, VTermPos *cursor_pos)
 
     ga_init2(&ga_text, 1, 90);
     ga_init2(&ga_cell, sizeof(cellattr_T), 90);
-    CLEAR_FIELD(cell);
-    CLEAR_FIELD(empty_cell);
+    clear_cellattr(&cell);
+    clear_cellattr(&empty_cell);
     cursor_pos->row = -1;
     cursor_pos->col = -1;
 
@@ -4853,7 +4907,7 @@ read_dump_file(FILE *fd, VTermPos *cursor_pos)
 			}
 			else if (c == '#')
 			{
-			    int red, green, blue, index = 0;
+			    int red, green, blue, index = 0, type;
 
 			    c = fgetc(fd);
 			    red = hex2nr(c);
@@ -4875,20 +4929,37 @@ read_dump_file(FILE *fd, VTermPos *cursor_pos)
 				index = index * 10 + (c - '0');
 				c = fgetc(fd);
 			    }
-
-			    if (is_bg)
+			    if (index == 0 || index == 255)
 			    {
-				cell.bg.red = red;
-				cell.bg.green = green;
-				cell.bg.blue = blue;
-				cell.bg.ansi_index = index;
+				type = VTERM_COLOR_RGB;
+				if (index == 0)
+				{
+				    if (is_bg)
+					type |= VTERM_COLOR_DEFAULT_BG;
+				    else
+					type |= VTERM_COLOR_DEFAULT_FG;
+				}
 			    }
 			    else
 			    {
+				type = VTERM_COLOR_INDEXED;
+				index -= 1;
+			    }
+			    if (is_bg)
+			    {
+				cell.bg.type = type;
+				cell.bg.red = red;
+				cell.bg.green = green;
+				cell.bg.blue = blue;
+				cell.bg.index = index;
+			    }
+			    else
+			    {
+				cell.fg.type = type;
 				cell.fg.red = red;
 				cell.fg.green = green;
 				cell.fg.blue = blue;
-				cell.fg.ansi_index = index;
+				cell.fg.index = index;
 			    }
 			}
 			else
@@ -5201,10 +5272,10 @@ term_load_dump(typval_T *argvars, typval_T *rettv, int do_diff)
 			if ((cellattr1 + col)->width
 						   != (cellattr2 + col)->width)
 			    textline[col] = 'w';
-			else if (!same_color(&(cellattr1 + col)->fg,
+			else if (!vterm_color_is_equal(&(cellattr1 + col)->fg,
 						   &(cellattr2 + col)->fg))
 			    textline[col] = 'f';
-			else if (!same_color(&(cellattr1 + col)->bg,
+			else if (!vterm_color_is_equal(&(cellattr1 + col)->bg,
 						   &(cellattr2 + col)->bg))
 			    textline[col] = 'b';
 			else if (vtermAttr2hl((cellattr1 + col)->attrs)
@@ -5783,6 +5854,7 @@ f_term_scrape(typval_T *argvars, typval_T *rettv)
 	else
 	{
 	    VTermScreenCell cell;
+
 	    if (vterm_screen_get_cell(screen, pos, &cell) == 0)
 		break;
 	    for (i = 0; i < VTERM_MAX_CHARS_PER_CELL; ++i)

src/testdir/test_channel.py

@@ -21,6 +21,9 @@ except ImportError:
 
 class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
 
+    def setup(self):
+        self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
     def handle(self):
         print("=== socket opened ===")
         while True:

src/testdir/test_json.vim

@@ -148,36 +148,36 @@ func Test_json_decode()
   call assert_fails("call json_decode('{\"\": \"ok\", \"\": \"bad\"}')", 'E938:')
 
   call assert_equal({'n': 1}, json_decode('{"n":1,}'))
-  call assert_fails("call json_decode(\"{'n':'1',}\")", 'E474:')
-  call assert_fails("call json_decode(\"'n'\")", 'E474:')
+  call assert_fails("call json_decode(\"{'n':'1',}\")", 'E491:')
+  call assert_fails("call json_decode(\"'n'\")", 'E491:')
 
-  call assert_fails('call json_decode("\"")', "E474:")
-  call assert_fails('call json_decode("blah")', "E474:")
+  call assert_fails('call json_decode("\"")', "E491:")
+  call assert_fails('call json_decode("blah")', "E491:")
   call assert_fails('call json_decode("true blah")', "E488:")
-  call assert_fails('call json_decode("<foobar>")', "E474:")
+  call assert_fails('call json_decode("<foobar>")', "E491:")
   call assert_fails('call json_decode("{\"a\":1,\"a\":2}")', "E938:")
 
-  call assert_fails('call json_decode("{")', "E474:")
-  call assert_fails('call json_decode("{foobar}")', "E474:")
-  call assert_fails('call json_decode("{\"n\",")', "E474:")
-  call assert_fails('call json_decode("{\"n\":")', "E474:")
-  call assert_fails('call json_decode("{\"n\":1")', "E474:")
-  call assert_fails('call json_decode("{\"n\":1,")', "E474:")
-  call assert_fails('call json_decode("{\"n\",1}")', "E474:")
-  call assert_fails('call json_decode("{-}")', "E474:")
+  call assert_fails('call json_decode("{")', "E491:")
+  call assert_fails('call json_decode("{foobar}")', "E491:")
+  call assert_fails('call json_decode("{\"n\",")', "E491:")
+  call assert_fails('call json_decode("{\"n\":")', "E491:")
+  call assert_fails('call json_decode("{\"n\":1")', "E491:")
+  call assert_fails('call json_decode("{\"n\":1,")', "E491:")
+  call assert_fails('call json_decode("{\"n\",1}")', "E491:")
+  call assert_fails('call json_decode("{-}")', "E491:")
 
-  call assert_fails('call json_decode("[foobar]")', "E474:")
-  call assert_fails('call json_decode("[")', "E474:")
-  call assert_fails('call json_decode("[1")', "E474:")
-  call assert_fails('call json_decode("[1,")', "E474:")
-  call assert_fails('call json_decode("[1 2]")', "E474:")
+  call assert_fails('call json_decode("[foobar]")', "E491:")
+  call assert_fails('call json_decode("[")', "E491:")
+  call assert_fails('call json_decode("[1")', "E491:")
+  call assert_fails('call json_decode("[1,")', "E491:")
+  call assert_fails('call json_decode("[1 2]")', "E491:")
 
-  call assert_fails('call json_decode("[1,,2]")', "E474:")
+  call assert_fails('call json_decode("[1,,2]")', "E491:")
 
-  call assert_fails('call json_decode("{{}:42}")', "E474:")
-  call assert_fails('call json_decode("{[]:42}")', "E474:")
+  call assert_fails('call json_decode("{{}:42}")', "E491:")
+  call assert_fails('call json_decode("{[]:42}")', "E491:")
 
-  call assert_fails('call json_decode("\"\\u111Z\"")', 'E474:')
+  call assert_fails('call json_decode("\"\\u111Z\"")', 'E491:')
   call assert_equal('[😂]', json_decode('"[\uD83D\uDE02]"'))
   call assert_equal('a😂b', json_decode('"a\uD83D\uDE02b"'))
 endfunc

src/testdir/test_lua.vim

@@ -33,7 +33,7 @@ func Test_lua_eval()
   " lua.eval with a number
   lua v = vim.eval('123')
   call assert_equal('number', luaeval('vim.type(v)'))
-  call assert_equal(123.0, luaeval('v'))
+  call assert_equal(123, luaeval('v'))
 
   " lua.eval with a string
   lua v = vim.eval('"abc"')
@@ -121,6 +121,18 @@ func Test_lua_window_line_col()
   bwipe!
 endfunc
 
+" Test vim.call
+func Test_lua_call()
+  call assert_equal(has('lua'), luaeval('vim.call("has", "lua")'))
+  call assert_equal(printf("Hello %s", "vim"), luaeval('vim.call("printf", "Hello %s", "vim")'))
+endfunc
+
+" Test vim.fn.*
+func Test_lua_fn()
+  call assert_equal(has('lua'), luaeval('vim.fn.has("lua")'))
+  call assert_equal(printf("Hello %s", "vim"), luaeval('vim.fn.printf("Hello %s", "vim")'))
+endfunc
+
 " Test setting the current window
 func Test_lua_window_set_current()
   new Xfoo1
@@ -252,7 +264,7 @@ endfunc
 func Test_lua_buffer_number_lines()
   new
   call setline(1, ['a', 'b', 'c'])
-  call assert_equal(3.0, luaeval('#vim.buffer()'))
+  call assert_equal(3, luaeval('#vim.buffer()'))
   bwipe!
 endfunc
 
@@ -311,15 +323,15 @@ func Test_lua_list()
   lua l:add(nil)
   lua l:add(vim.eval("[1, 2, 3]"))
   lua l:add(vim.eval("{'a':1, 'b':2, 'c':3}"))
-  call assert_equal([123.0, 'abc', v:true, v:false, v:null, [1, 2, 3], {'a': 1, 'b': 2, 'c': 3}], l)
-  call assert_equal(7.0, luaeval('#l'))
+  call assert_equal([123, 'abc', v:true, v:false, v:null, [1, 2, 3], {'a': 1, 'b': 2, 'c': 3}], l)
+  call assert_equal(7, luaeval('#l'))
   call assert_match('^list: \%(0x\)\?\x\+$', luaeval('tostring(l)'))
 
   lua l[0] = 124
   lua l[5] = nil
   lua l:insert('first')
   lua l:insert('xx', 3)
-  call assert_equal(['first', 124.0, 'abc', 'xx', v:true, v:false, v:null, {'a': 1, 'b': 2, 'c': 3}], l)
+  call assert_equal(['first', 124, 'abc', 'xx', v:true, v:false, v:null, {'a': 1, 'b': 2, 'c': 3}], l)
 
   lockvar 1 l
   call assert_fails('lua l:add("x")', '[string "vim chunk"]:1: list is locked')
@@ -355,16 +367,16 @@ func Test_lua_recursive_list()
   lua l = vim.list():add(1):add(2)
   lua l = l:add(l)
 
-  call assert_equal(1.0, luaeval('l[0]'))
-  call assert_equal(2.0, luaeval('l[1]'))
+  call assert_equal(1, luaeval('l[0]'))
+  call assert_equal(2, luaeval('l[1]'))
 
-  call assert_equal(1.0, luaeval('l[2][0]'))
-  call assert_equal(2.0, luaeval('l[2][1]'))
+  call assert_equal(1, luaeval('l[2][0]'))
+  call assert_equal(2, luaeval('l[2][1]'))
 
-  call assert_equal(1.0, luaeval('l[2][2][0]'))
-  call assert_equal(2.0, luaeval('l[2][2][1]'))
+  call assert_equal(1, luaeval('l[2][2][0]'))
+  call assert_equal(2, luaeval('l[2][2][1]'))
 
-  call assert_equal('[1.0, 2.0, [...]]', string(luaeval('l')))
+  call assert_equal('[1, 2, [...]]', string(luaeval('l')))
 
   call assert_match('^list: \%(0x\)\?\x\+$', luaeval('tostring(l)'))
   call assert_equal(luaeval('tostring(l)'), luaeval('tostring(l[2])'))
@@ -386,15 +398,15 @@ func Test_lua_dict()
   lua d[3] = false
   lua d[4] = vim.eval("[1, 2, 3]")
   lua d[5] = vim.eval("{'a':1, 'b':2, 'c':3}")
-  call assert_equal({'0':123.0, '1':'abc', '2':v:true, '3':v:false, '4': [1, 2, 3], '5': {'a':1, 'b':2, 'c':3}}, d)
-  call assert_equal(6.0, luaeval('#d'))
+  call assert_equal({'0':123, '1':'abc', '2':v:true, '3':v:false, '4': [1, 2, 3], '5': {'a':1, 'b':2, 'c':3}}, d)
+  call assert_equal(6, luaeval('#d'))
   call assert_match('^dict: \%(0x\)\?\x\+$', luaeval('tostring(d)'))
 
   call assert_equal('abc', luaeval('d[1]'))
 
   lua d[0] = 124
   lua d[4] = nil
-  call assert_equal({'0':124.0, '1':'abc', '2':v:true, '3':v:false, '5': {'a':1, 'b':2, 'c':3}}, d)
+  call assert_equal({'0':124, '1':'abc', '2':v:true, '3':v:false, '5': {'a':1, 'b':2, 'c':3}}, d)
 
   lockvar 1 d
   call assert_fails('lua d[6] = 1', '[string "vim chunk"]:1: dict is locked')
@@ -441,16 +453,16 @@ func Test_lua_blob()
 
   lua b = vim.blob("\x00\x00\x00\x00")
   call assert_equal(0z00000000, luaeval('b'))
-  call assert_equal(4.0, luaeval('#b'))
+  call assert_equal(4, luaeval('#b'))
   lua b[0], b[1], b[2], b[3] = 1, 32, 256, 0xff
   call assert_equal(0z012000ff, luaeval('b'))
   lua b[4] = string.byte("z", 1)
   call assert_equal(0z012000ff.7a, luaeval('b'))
-  call assert_equal(5.0, luaeval('#b'))
+  call assert_equal(5, luaeval('#b'))
   call assert_fails('lua b[#b+1] = 0x80', '[string "vim chunk"]:1: index out of range')
   lua b:add("12ab")
   call assert_equal(0z012000ff.7a313261.62, luaeval('b'))
-  call assert_equal(9.0, luaeval('#b'))
+  call assert_equal(9, luaeval('#b'))
   call assert_fails('lua b:add(nil)', '[string "vim chunk"]:1: string expected, got nil')
   call assert_fails('lua b:add(true)', '[string "vim chunk"]:1: string expected, got boolean')
   call assert_fails('lua b:add({})', '[string "vim chunk"]:1: string expected, got table')
@@ -552,12 +564,12 @@ endfunc
 " Test :luafile foo.lua
 func Test_luafile()
   call delete('Xlua_file')
-  call writefile(["str = 'hello'", "num = 123.0" ], 'Xlua_file')
+  call writefile(["str = 'hello'", "num = 123" ], 'Xlua_file')
   call setfperm('Xlua_file', 'r-xr-xr-x')
 
   luafile Xlua_file
   call assert_equal('hello', luaeval('str'))
-  call assert_equal(123.0, luaeval('num'))
+  call assert_equal(123, luaeval('num'))
 
   lua str, num = nil
   call delete('Xlua_file')

src/testdir/test_netbeans.vim

@@ -609,7 +609,7 @@ func Nb_basic(port)
 
   " detach
   call appendbufline(cmdbufnr, '$', 'detach_Test')
-  call WaitFor('len(readfile("Xnetbeans")) >= (g:last + 6)')
+  call WaitFor('len(readfile("Xnetbeans")) >= (g:last + 8)')
   call WaitForAssert({-> assert_equal('0:disconnect=91', readfile("Xnetbeans")[-1])})
 
   " the connection was closed

src/testdir/test_popupwin.vim

@@ -2426,10 +2426,10 @@ func Test_popupwin_terminal_buffer()
   let g:test_is_flaky = 1
 
   let origwin = win_getid()
-  let ptybuf = term_start(&shell, #{hidden: 1})
-  let winid = popup_create(ptybuf, #{minwidth: 40, minheight: 10})
+  let termbuf = term_start(&shell, #{hidden: 1})
+  let winid = popup_create(termbuf, #{minwidth: 40, minheight: 10})
   " Wait for shell to start
-  call WaitForAssert({-> assert_equal("run", job_status(term_getjob(ptybuf)))})
+  call WaitForAssert({-> assert_equal("run", job_status(term_getjob(termbuf)))})
   sleep 100m
   " Check this doesn't crash
   call assert_equal(winnr(), winnr('j'))
@@ -2440,11 +2440,16 @@ func Test_popupwin_terminal_buffer()
   " Cannot quit while job is running
   call assert_fails('call feedkeys("\<C-W>:quit\<CR>", "xt")', 'E948:')
 
-  " Cannot enter Terminal-Normal mode.
+  " Cannot enter Terminal-Normal mode. (TODO: but it works...)
   call feedkeys("xxx\<C-W>N", 'xt')
   call assert_fails('call feedkeys("gf", "xt")', 'E863:')
   call feedkeys("a\<C-U>", 'xt')
 
+  " Cannot open a second one.
+  let termbuf2 = term_start(&shell, #{hidden: 1})
+  call assert_fails('call popup_create(termbuf2, #{})', 'E861:')
+  call term_sendkeys(termbuf2, "exit\<CR>")
+
   " Exiting shell closes popup window
   call feedkeys("exit\<CR>", 'xt')
   " Wait for shell to exit

src/testdir/test_terminal.vim

@@ -938,10 +938,10 @@ func Test_terminal_composing_unicode()
   call WaitForAssert({-> assert_equal(txt, term_getline(buf, lnum[1] + 1))}, 1000)
   let l = term_scrape(buf, lnum[1] + 1)
   call assert_equal("\u304b\u3099", l[0].chars)
-  call assert_equal("\u304e", l[1].chars)
-  call assert_equal("\u304f\u3099", l[2].chars)
-  call assert_equal("\u3052", l[3].chars)
-  call assert_equal("\u3053\u3099", l[4].chars)
+  call assert_equal("\u304e", l[2].chars)
+  call assert_equal("\u304f\u3099", l[3].chars)
+  call assert_equal("\u3052", l[5].chars)
+  call assert_equal("\u3053\u3099", l[6].chars)
 
   " \u00a0 + composing
   let txt = "abc\u00a0\u0308"
@@ -2587,9 +2587,8 @@ func Test_double_popup_terminal()
   let buf1 = term_start(&shell, #{hidden: 1})
   let win1 = popup_create(buf1, {})
   let buf2 = term_start(&shell, #{hidden: 1})
-  let win2 = popup_create(buf2, {})
+  call assert_fails('call popup_create(buf2, {})', 'E861:')
   call popup_close(win1)
-  call popup_close(win2)
   exe buf1 .. 'bwipe!'
   exe buf2 .. 'bwipe!'
 endfunc
@@ -2619,10 +2618,8 @@ func Test_term_nasty_callback()
   CheckExecutable sh
 
   set hidden
-  let g:buf0 = term_start('sh', #{hidden: 1})
+  let g:buf0 = term_start('sh', #{hidden: 1, term_finish: 'close'})
   call popup_create(g:buf0, {})
-  let g:buf1 = term_start('sh', #{hidden: 1, term_finish: 'close'})
-  call popup_create(g:buf1, {})
   call assert_fails("call term_start(['sh', '-c'], #{curwin: 1})", 'E863:')
 
   call popup_clear(1)

src/testdir/test_vim9_expr.vim

@@ -153,9 +153,13 @@ let adict = #{aaa: 2, bbb: 8}
 
 " test == comperator
 def Test_expr4_equal()
+  let trueVar = true
+  let falseVar = false
   assert_equal(true, true == true)
   assert_equal(false, true ==
 			false)
+  assert_equal(true, true == trueVar)
+  assert_equal(false, true == falseVar)
   assert_equal(true, true == g:atrue)
   assert_equal(false, g:atrue == false)
 
@@ -164,8 +168,12 @@ def Test_expr4_equal()
   assert_equal(true, g:anone == v:none)
   assert_equal(false, v:none == g:anull)
 
+  let nr0 = 0
+  let nr61 = 61
   assert_equal(false, 2 == 0)
+  assert_equal(false, 2 == nr0)
   assert_equal(true, 61 == 61)
+  assert_equal(true, 61 == nr61)
   assert_equal(true, g:anint == 10)
   assert_equal(false, 61 == g:anint)
 
@@ -237,9 +245,13 @@ enddef
 
 " test != comperator
 def Test_expr4_notequal()
+  let trueVar = true
+  let falseVar = false
   assert_equal(false, true != true)
   assert_equal(true, true !=
 			false)
+  assert_equal(false, true != trueVar)
+  assert_equal(true, true != falseVar)
   assert_equal(false, true != g:atrue)
   assert_equal(true, g:atrue != false)
 
@@ -248,8 +260,12 @@ def Test_expr4_notequal()
   assert_equal(false, g:anone != v:none)
   assert_equal(true, v:none != g:anull)
 
+  let nr55 = 55
+  let nr0 = 55
   assert_equal(true, 2 != 0)
+  assert_equal(true, 2 != nr0)
   assert_equal(false, 55 != 55)
+  assert_equal(false, 55 != nr55)
   assert_equal(false, g:anint != 10)
   assert_equal(true, 61 != g:anint)
 
@@ -313,6 +329,12 @@ def Test_expr4_greater()
 		1)
   assert_false(2 > 2)
   assert_false(2 > 3)
+  let nr2 = 2
+  assert_true(nr2 > 0)
+  assert_true(nr2 >
+		1)
+  assert_false(nr2 > 2)
+  assert_false(nr2 > 3)
   if has('float')
     let ff = 2.0
     assert_true(ff > 0.0)
@@ -328,6 +350,10 @@ def Test_expr4_greaterequal()
   assert_true(2 >=
 			2)
   assert_false(2 >= 3)
+  let nr2 = 2
+  assert_true(nr2 >= 0)
+  assert_true(nr2 >= 2)
+  assert_false(nr2 >= 3)
   if has('float')
     let ff = 2.0
     assert_true(ff >= 0.0)
@@ -342,6 +368,10 @@ def Test_expr4_smaller()
   assert_false(2 <
 			2)
   assert_true(2 < 3)
+  let nr2 = 2
+  assert_false(nr2 < 0)
+  assert_false(nr2 < 2)
+  assert_true(nr2 < 3)
   if has('float')
     let ff = 2.0
     assert_false(ff < 0.0)
@@ -357,6 +387,11 @@ def Test_expr4_smallerequal()
 			1)
   assert_true(2 <= 2)
   assert_true(2 <= 3)
+  let nr2 = 2
+  assert_false(nr2 <= 0)
+  assert_false(nr2 <= 1)
+  assert_true(nr2 <= 2)
+  assert_true(nr2 <= 3)
   if has('float')
     let ff = 2.0
     assert_false(ff <= 0.0)

src/testdir/test_vim9_script.vim

@@ -23,21 +23,6 @@ def Test_assignment()
   let bool2: bool = false
   assert_equal(v:false, bool2)
 
-  let list1: list<bool> = [false, true, false]
-  let list2: list<number> = [1, 2, 3]
-  let list3: list<string> = ['sdf', 'asdf']
-  let list4: list<any> = ['yes', true, 1234]
-  let list5: list<blob> = [0z01, 0z02]
-
-  let listS: list<string> = []
-  let listN: list<number> = []
-
-  let dict1: dict<bool> = #{one: false, two: true}
-  let dict2: dict<number> = #{one: 1, two: 2}
-  let dict3: dict<string> = #{key: 'value'}
-  let dict4: dict<any> = #{one: 1, two: '2'}
-  let dict5: dict<blob> = #{one: 0z01, tw: 0z02}
-
   call CheckDefFailure(['let x:string'], 'E1069:')
   call CheckDefFailure(['let x:string = "x"'], 'E1069:')
   call CheckDefFailure(['let a:string = "x"'], 'E1069:')
@@ -57,11 +42,6 @@ def Test_assignment()
   let Funky2: func = function('len')
   let Party2: func = funcref('g:Test_syntax')
 
-  # type becomes list<any>
-  let somelist = rand() > 0 ? [1, 2, 3] : ['a', 'b', 'c']
-  # type becomes dict<any>
-  let somedict = rand() > 0 ? #{a: 1, b: 2} : #{a: 'a', b: 'b'}
-
   g:newvar = 'new'
   assert_equal('new', g:newvar)
 
@@ -128,6 +108,42 @@ def Test_assignment()
   call CheckDefFailure(['v:errmsg += 123'], 'E1013:')
 enddef
 
+def Test_assignment_list()
+  let list1: list<bool> = [false, true, false]
+  let list2: list<number> = [1, 2, 3]
+  let list3: list<string> = ['sdf', 'asdf']
+  let list4: list<any> = ['yes', true, 1234]
+  let list5: list<blob> = [0z01, 0z02]
+
+  let listS: list<string> = []
+  let listN: list<number> = []
+
+  assert_equal([1, 2, 3], list2)
+  list2[-1] = 99
+  assert_equal([1, 2, 99], list2)
+  list2[-2] = 88
+  assert_equal([1, 88, 99], list2)
+  list2[-3] = 77
+  assert_equal([77, 88, 99], list2)
+  call CheckDefExecFailure(['let ll = [1, 2, 3]', 'll[-4] = 6'], 'E684:')
+
+  # type becomes list<any>
+  let somelist = rand() > 0 ? [1, 2, 3] : ['a', 'b', 'c']
+enddef
+
+def Test_assignment_dict()
+  let dict1: dict<bool> = #{one: false, two: true}
+  let dict2: dict<number> = #{one: 1, two: 2}
+  let dict3: dict<string> = #{key: 'value'}
+  let dict4: dict<any> = #{one: 1, two: '2'}
+  let dict5: dict<blob> = #{one: 0z01, tw: 0z02}
+
+  call CheckDefExecFailure(['let dd = {}', 'dd[""] = 6'], 'E713:')
+
+  # type becomes dict<any>
+  let somedict = rand() > 0 ? #{a: 1, b: 2} : #{a: 'a', b: 'b'}
+enddef
+
 def Test_assignment_local()
   " Test in a separated file in order not to the current buffer/window/tab is
   " changed.
@@ -1736,6 +1752,8 @@ def Test_let_func_call()
       return 'this'
     endfunc
     let val: string = GetValue() 
+    " env var is always a string
+    let env = $TERM
   END
   writefile(lines, 'Xfinished')
   source Xfinished
@@ -1752,13 +1770,40 @@ def Test_let_missing_type()
     func GetValue()
       return 'this'
     endfunc
-    let val = GetValue() 
+    let val = GetValue()
   END
   CheckScriptFailure(lines, 'E1091:')
 
   lines =<< trim END
     vim9script
-    let var = g:unkown
+    func GetValue()
+      return 'this'
+    endfunc
+    let val = [GetValue()]
+  END
+  CheckScriptFailure(lines, 'E1091:')
+
+  lines =<< trim END
+    vim9script
+    func GetValue()
+      return 'this'
+    endfunc
+    let val = {GetValue(): 123}
+  END
+  CheckScriptFailure(lines, 'E1091:')
+
+  lines =<< trim END
+    vim9script
+    func GetValue()
+      return 'this'
+    endfunc
+    let val = {'a': GetValue()}
+  END
+  CheckScriptFailure(lines, 'E1091:')
+
+  lines =<< trim END
+    vim9script
+    let var = g:unknown
   END
   CheckScriptFailure(lines, 'E1091:')
 

src/version.c

@@ -746,6 +746,72 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    804,
+/**/
+    803,
+/**/
+    802,
+/**/
+    801,
+/**/
+    800,
+/**/
+    799,
+/**/
+    798,
+/**/
+    797,
+/**/
+    796,
+/**/
+    795,
+/**/
+    794,
+/**/
+    793,
+/**/
+    792,
+/**/
+    791,
+/**/
+    790,
+/**/
+    789,
+/**/
+    788,
+/**/
+    787,
+/**/
+    786,
+/**/
+    785,
+/**/
+    784,
+/**/
+    783,
+/**/
+    782,
+/**/
+    781,
+/**/
+    780,
+/**/
+    779,
+/**/
+    778,
+/**/
+    777,
+/**/
+    776,
+/**/
+    775,
+/**/
+    774,
+/**/
+    773,
+/**/
+    772,
 /**/
     771,
 /**/