Update Scintilla to version 3.10.2

Closes #971, closes #1947.

scintilla/gtk/ScintillaGTK.cxx

@@ -1075,9 +1075,10 @@ void ScintillaGTK::NotifyChange() {
 }
 
 void ScintillaGTK::NotifyFocus(bool focus) {
-	g_signal_emit(G_OBJECT(sci), scintilla_signals[COMMAND_SIGNAL], 0,
-	                Platform::LongFromTwoShorts
-					(GetCtrlID(), focus ? SCEN_SETFOCUS : SCEN_KILLFOCUS), PWidget(wMain));
+	if (commandEvents)
+		g_signal_emit(G_OBJECT(sci), scintilla_signals[COMMAND_SIGNAL], 0,
+				Platform::LongFromTwoShorts
+						(GetCtrlID(), focus ? SCEN_SETFOCUS : SCEN_KILLFOCUS), PWidget(wMain));
 	Editor::NotifyFocus(focus);
 }
 

scintilla/gtk/ScintillaGTKAccessible.cxx

@@ -1128,7 +1128,7 @@ AtkObject *ScintillaGTKAccessible::WidgetGetAccessibleImpl(GtkWidget *widget, At
 	if (parent_atk_type == 0) {
 		AtkObject *parent_obj = GTK_WIDGET_CLASS(widget_parent_class)->get_accessible(widget);
 		if (parent_obj) {
-			GType parent_atk_type = G_OBJECT_TYPE(parent_obj);
+			parent_atk_type = G_OBJECT_TYPE(parent_obj);
 
 			// Figure out whether accessibility is enabled by looking at the type of the accessible
 			// object which would be created for the parent type of ScintillaObject.

scintilla/include/Platform.h

@@ -25,6 +25,7 @@
 #define PLAT_FOX 0
 #define PLAT_CURSES 0
 #define PLAT_TK 0
+#define PLAT_HAIKU 0
 
 #if defined(FOX)
 #undef PLAT_FOX
@@ -38,6 +39,10 @@
 #undef PLAT_CURSES
 #define PLAT_CURSES 1
 
+#elif defined(__HAIKU__)
+#undef PLAT_HAIKU
+#define PLAT_HAIKU 1
+
 #elif defined(SCINTILLA_QT)
 #undef PLAT_QT
 #define PLAT_QT 1

scintilla/include/SciLexer.h

@@ -136,6 +136,9 @@
 #define SCLEX_EDIFACT 121
 #define SCLEX_INDENT 122
 #define SCLEX_MAXIMA 123
+#define SCLEX_STATA 124
+#define SCLEX_SAS 125
+#define SCLEX_NIM 126
 #define SCLEX_LPEG 999
 #define SCLEX_AUTOMATIC 1000
 #define SCE_P_DEFAULT 0
@@ -1826,6 +1829,51 @@
 #define SCE_EDI_UNA 6
 #define SCE_EDI_UNH 7
 #define SCE_EDI_BADSEGMENT 8
+#define SCE_STATA_DEFAULT 0
+#define SCE_STATA_COMMENT 1
+#define SCE_STATA_COMMENTLINE 2
+#define SCE_STATA_COMMENTBLOCK 3
+#define SCE_STATA_NUMBER 4
+#define SCE_STATA_OPERATOR 5
+#define SCE_STATA_IDENTIFIER 6
+#define SCE_STATA_STRING 7
+#define SCE_STATA_TYPE 8
+#define SCE_STATA_WORD 9
+#define SCE_STATA_GLOBAL_MACRO 10
+#define SCE_STATA_MACRO 11
+#define SCE_SAS_DEFAULT 0
+#define SCE_SAS_COMMENT 1
+#define SCE_SAS_COMMENTLINE 2
+#define SCE_SAS_COMMENTBLOCK 3
+#define SCE_SAS_NUMBER 4
+#define SCE_SAS_OPERATOR 5
+#define SCE_SAS_IDENTIFIER 6
+#define SCE_SAS_STRING 7
+#define SCE_SAS_TYPE 8
+#define SCE_SAS_WORD 9
+#define SCE_SAS_GLOBAL_MACRO 10
+#define SCE_SAS_MACRO 11
+#define SCE_SAS_MACRO_KEYWORD 12
+#define SCE_SAS_BLOCK_KEYWORD 13
+#define SCE_SAS_MACRO_FUNCTION 14
+#define SCE_SAS_STATEMENT 15
+#define SCE_NIM_DEFAULT 0
+#define SCE_NIM_COMMENT 1
+#define SCE_NIM_COMMENTDOC 2
+#define SCE_NIM_COMMENTLINE 3
+#define SCE_NIM_COMMENTLINEDOC 4
+#define SCE_NIM_NUMBER 5
+#define SCE_NIM_STRING 6
+#define SCE_NIM_CHARACTER 7
+#define SCE_NIM_WORD 8
+#define SCE_NIM_TRIPLE 9
+#define SCE_NIM_TRIPLEDOUBLE 10
+#define SCE_NIM_BACKTICKS 11
+#define SCE_NIM_FUNCNAME 12
+#define SCE_NIM_STRINGEOL 13
+#define SCE_NIM_NUMERROR 14
+#define SCE_NIM_OPERATOR 15
+#define SCE_NIM_IDENTIFIER 16
 /* --Autogenerated -- end of section automatically generated from Scintilla.iface */
 
 #endif

scintilla/include/Scintilla.h

@@ -365,6 +365,7 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SCI_GETLINEINDENTPOSITION 2128
 #define SCI_GETCOLUMN 2129
 #define SCI_COUNTCHARACTERS 2633
+#define SCI_COUNTCODEUNITS 2715
 #define SCI_SETHSCROLLBAR 2130
 #define SCI_GETHSCROLLBAR 2131
 #define SC_IV_NONE 0
@@ -698,6 +699,8 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SCI_RELEASEDOCUMENT 2377
 #define SCI_GETDOCUMENTOPTIONS 2379
 #define SCI_GETMODEVENTMASK 2378
+#define SCI_SETCOMMANDEVENTS 2717
+#define SCI_GETCOMMANDEVENTS 2718
 #define SCI_SETFOCUS 2380
 #define SCI_GETFOCUS 2381
 #define SC_STATUS_OK 0
@@ -755,6 +758,7 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SCI_POSITIONBEFORE 2417
 #define SCI_POSITIONAFTER 2418
 #define SCI_POSITIONRELATIVE 2670
+#define SCI_POSITIONRELATIVECODEUNITS 2716
 #define SCI_COPYRANGE 2419
 #define SCI_COPYTEXT 2420
 #define SC_SEL_STREAM 0
@@ -1108,6 +1112,16 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SCN_AUTOCCOMPLETED 2030
 #define SCN_MARGINRIGHTCLICK 2031
 #define SCN_AUTOCSELECTIONCHANGE 2032
+#ifndef SCI_DISABLE_PROVISIONAL
+#define SC_LINECHARACTERINDEX_NONE 0
+#define SC_LINECHARACTERINDEX_UTF32 1
+#define SC_LINECHARACTERINDEX_UTF16 2
+#define SCI_GETLINECHARACTERINDEX 2710
+#define SCI_ALLOCATELINECHARACTERINDEX 2711
+#define SCI_RELEASELINECHARACTERINDEX 2712
+#define SCI_LINEFROMINDEXPOSITION 2713
+#define SCI_INDEXPOSITIONFROMLINE 2714
+#endif
 /* --Autogenerated -- end of section automatically generated from Scintilla.iface */
 
 /* These structures are defined to be exactly the same shape as the Win32

scintilla/include/Scintilla.iface

@@ -862,6 +862,9 @@ get int GetColumn=2129(position pos,)
 # Count characters between two positions.
 fun int CountCharacters=2633(position start, position end)
 
+# Count code units between two positions.
+fun int CountCodeUnits=2715(position start, position end)
+
 # Show or hide the horizontal scroll bar.
 set void SetHScrollBar=2130(bool visible,)
 # Is the horizontal scroll bar visible?
@@ -1803,6 +1806,12 @@ get int GetDocumentOptions=2379(,)
 # Get which document modification events are sent to the container.
 get int GetModEventMask=2378(,)
 
+# Set whether command events are sent to the container.
+set void SetCommandEvents=2717(bool commandEvents,)
+
+# Get whether command events are sent to the container.
+get bool GetCommandEvents=2718(,)
+
 # Change internal focus flag.
 set void SetFocus=2380(bool focus,)
 # Get internal focus flag.
@@ -1966,6 +1975,11 @@ fun position PositionAfter=2418(position pos,)
 # of characters. Returned value is always between 0 and last position in document.
 fun position PositionRelative=2670(position pos, int relative)
 
+# Given a valid document position, return a position that differs in a number
+# of UTF-16 code units. Returned value is always between 0 and last position in document.
+# The result may point half way (2 bytes) inside a non-BMP character.
+fun position PositionRelativeCodeUnits=2716(position pos, int relative)
+
 # Copy a range of text to the clipboard. Positions are clipped into the document.
 fun void CopyRange=2419(position start, position end)
 
@@ -2939,6 +2953,9 @@ val SCLEX_JSON=120
 val SCLEX_EDIFACT=121
 val SCLEX_INDENT=122
 val SCLEX_MAXIMA=123
+val SCLEX_STATA=124
+val SCLEX_SAS=125
+val SCLEX_NIM=126
 val SCLEX_LPEG=999
 
 # When a lexer specifies its language as SCLEX_AUTOMATIC it receives a
@@ -4861,6 +4878,57 @@ val SCE_EDI_SEP_RELEASE=5
 val SCE_EDI_UNA=6
 val SCE_EDI_UNH=7
 val SCE_EDI_BADSEGMENT=8
+# Lexical states for SCLEX_STATA
+lex STATA=SCLEX_STATA SCE_STATA_
+val SCE_STATA_DEFAULT=0
+val SCE_STATA_COMMENT=1
+val SCE_STATA_COMMENTLINE=2
+val SCE_STATA_COMMENTBLOCK=3
+val SCE_STATA_NUMBER=4
+val SCE_STATA_OPERATOR=5
+val SCE_STATA_IDENTIFIER=6
+val SCE_STATA_STRING=7
+val SCE_STATA_TYPE=8
+val SCE_STATA_WORD=9
+val SCE_STATA_GLOBAL_MACRO=10
+val SCE_STATA_MACRO=11
+# Lexical states for SCLEX_SAS
+lex SAS=SCLEX_SAS SCE_SAS_
+val SCE_SAS_DEFAULT=0
+val SCE_SAS_COMMENT=1
+val SCE_SAS_COMMENTLINE=2
+val SCE_SAS_COMMENTBLOCK=3
+val SCE_SAS_NUMBER=4
+val SCE_SAS_OPERATOR=5
+val SCE_SAS_IDENTIFIER=6
+val SCE_SAS_STRING=7
+val SCE_SAS_TYPE=8
+val SCE_SAS_WORD=9
+val SCE_SAS_GLOBAL_MACRO=10
+val SCE_SAS_MACRO=11
+val SCE_SAS_MACRO_KEYWORD=12
+val SCE_SAS_BLOCK_KEYWORD=13
+val SCE_SAS_MACRO_FUNCTION=14
+val SCE_SAS_STATEMENT=15
+# Lexical states for SCLEX_NIM 
+lex Nim=SCLEX_NIM SCE_NIM_
+val SCE_NIM_DEFAULT=0
+val SCE_NIM_COMMENT=1
+val SCE_NIM_COMMENTDOC=2
+val SCE_NIM_COMMENTLINE=3
+val SCE_NIM_COMMENTLINEDOC=4
+val SCE_NIM_NUMBER=5
+val SCE_NIM_STRING=6
+val SCE_NIM_CHARACTER=7
+val SCE_NIM_WORD=8
+val SCE_NIM_TRIPLE=9
+val SCE_NIM_TRIPLEDOUBLE=10
+val SCE_NIM_BACKTICKS=11
+val SCE_NIM_FUNCNAME=12
+val SCE_NIM_STRINGEOL=13
+val SCE_NIM_NUMERROR=14
+val SCE_NIM_OPERATOR=15
+val SCE_NIM_IDENTIFIER=16
 
 # Events
 
@@ -4898,10 +4966,28 @@ evt void AutoCCompleted=2030(string text, int position, int ch, CompletionMethod
 evt void MarginRightClick=2031(int modifiers, int position, int margin)
 evt void AutoCSelectionChange=2032(int listType, string text, int position)
 
-# There are no provisional APIs currently.
-
 cat Provisional
 
+enu LineCharacterIndexType=SC_LINECHARACTERINDEX_
+val SC_LINECHARACTERINDEX_NONE=0
+val SC_LINECHARACTERINDEX_UTF32=1
+val SC_LINECHARACTERINDEX_UTF16=2
+
+# Retrieve line character index state.
+get int GetLineCharacterIndex=2710(,)
+
+# Request line character index be created or its use count increased.
+fun void AllocateLineCharacterIndex=2711(int lineCharacterIndex,)
+
+# Decrease use count of line character index and remove if 0.
+fun void ReleaseLineCharacterIndex=2712(int lineCharacterIndex,)
+
+# Retrieve the document line containing a position measured in index units.
+fun int LineFromIndexPosition=2713(position pos, int lineCharacterIndex)
+
+# Retrieve the position measured in index units at the start of a document line.
+fun position IndexPositionFromLine=2714(int line, int lineCharacterIndex)
+
 cat Deprecated
 
 # Divide each styling byte into lexical class bits (default: 5) and indicator

scintilla/lexers/LexBash.cxx

@@ -12,16 +12,23 @@
 #include <stdarg.h>
 #include <assert.h>
 
+#include <string>
+#include <vector>
+#include <map>
+
 #include "ILexer.h"
 #include "Scintilla.h"
 #include "SciLexer.h"
 
+#include "StringCopy.h"
 #include "WordList.h"
 #include "LexAccessor.h"
-#include "Accessor.h"
 #include "StyleContext.h"
 #include "CharacterSet.h"
 #include "LexerModule.h"
+#include "OptionSet.h"
+#include "SubStyles.h"
+#include "DefaultLexer.h"
 
 using namespace Scintilla;
 
@@ -58,7 +65,9 @@ using namespace Scintilla;
 
 #define BASH_DELIM_STACK_MAX	7
 
-static inline int translateBashDigit(int ch) {
+namespace {
+
+inline int translateBashDigit(int ch) {
 	if (ch >= '0' && ch <= '9') {
 		return ch - '0';
 	} else if (ch >= 'a' && ch <= 'z') {
@@ -73,7 +82,7 @@ static inline int translateBashDigit(int ch) {
 	return BASH_BASE_ERROR;
 }
 
-static inline int getBashNumberBase(char *s) {
+inline int getBashNumberBase(char *s) {
 	int i = 0;
 	int base = 0;
 	while (*s) {
@@ -86,7 +95,7 @@ static inline int getBashNumberBase(char *s) {
 	return base;
 }
 
-static int opposite(int ch) {
+int opposite(int ch) {
 	if (ch == '(') return ')';
 	if (ch == '[') return ']';
 	if (ch == '{') return '}';
@@ -94,7 +103,7 @@ static int opposite(int ch) {
 	return ch;
 }
 
-static int GlobScan(StyleContext &sc) {
+int GlobScan(StyleContext &sc) {
 	// forward scan for zsh globs, disambiguate versus bash arrays
 	// complex expressions may still fail, e.g. unbalanced () '' "" etc
 	int c, sLen = 0;
@@ -120,10 +129,171 @@ static int GlobScan(StyleContext &sc) {
 	return 0;
 }
 
-static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int initStyle,
-							 WordList *keywordlists[], Accessor &styler) {
+bool IsCommentLine(Sci_Position line, LexAccessor &styler) {
+	Sci_Position pos = styler.LineStart(line);
+	Sci_Position eol_pos = styler.LineStart(line + 1) - 1;
+	for (Sci_Position i = pos; i < eol_pos; i++) {
+		char ch = styler[i];
+		if (ch == '#')
+			return true;
+		else if (ch != ' ' && ch != '\t')
+			return false;
+	}
+	return false;
+}
 
-	WordList &keywords = *keywordlists[0];
+struct OptionsBash {
+	bool fold;
+	bool foldComment;
+	bool foldCompact;
+
+	OptionsBash() {
+		fold = false;
+		foldComment = false;
+		foldCompact = true;
+	}
+};
+
+const char * const bashWordListDesc[] = {
+	"Keywords",
+	0
+};
+
+struct OptionSetBash : public OptionSet<OptionsBash> {
+	OptionSetBash() {
+		DefineProperty("fold", &OptionsBash::fold);
+
+		DefineProperty("fold.comment", &OptionsBash::foldComment);
+
+		DefineProperty("fold.compact", &OptionsBash::foldCompact);
+
+		DefineWordListSets(bashWordListDesc);
+	}
+};
+
+const char styleSubable[] = { SCE_SH_IDENTIFIER, SCE_SH_SCALAR, 0 };
+
+LexicalClass lexicalClasses[] = {
+	// Lexer Bash SCLEX_BASH SCE_SH_:
+	0, "SCE_SH_DEFAULT", "default", "White space",
+	1, "SCE_SH_ERROR", "error", "Error",
+	2, "SCE_SH_COMMENTLINE", "comment line", "Line comment: #",
+	3, "SCE_SH_NUMBER", "literal numeric", "Number",
+	4, "SCE_SH_WORD", "keyword", "Keyword",
+	5, "SCE_SH_STRING", "literal string", "String",
+	6, "SCE_SH_CHARACTER", "literal string", "Single quoted string",
+	7, "SCE_SH_OPERATOR", "operator", "Operators",
+	8, "SCE_SH_IDENTIFIER", "identifier", "Identifiers",
+	9, "SCE_SH_SCALAR", "identifier", "Scalar variable",
+	10, "SCE_SH_PARAM", "identifier", "Parameter",
+	11, "SCE_SH_BACKTICKS", "literal string", "Backtick quoted command",
+	12, "SCE_SH_HERE_DELIM", "operator", "Heredoc delimiter",
+	13, "SCE_SH_HERE_Q", "literal string", "Heredoc quoted string",
+};
+
+}
+
+class LexerBash : public DefaultLexer {
+	WordList keywords;
+	OptionsBash options;
+	OptionSetBash osBash;
+	enum { ssIdentifier, ssScalar };
+	SubStyles subStyles;
+public:
+	LexerBash() :
+		DefaultLexer(lexicalClasses, ELEMENTS(lexicalClasses)),
+		subStyles(styleSubable, 0x80, 0x40, 0) {
+	}
+	virtual ~LexerBash() {
+	}
+	void SCI_METHOD Release() override {
+		delete this;
+	}
+	int SCI_METHOD Version() const override {
+		return lvMetaData;
+	}
+	const char * SCI_METHOD PropertyNames() override {
+		return osBash.PropertyNames();
+	}
+	int SCI_METHOD PropertyType(const char* name) override {
+		return osBash.PropertyType(name);
+	}
+	const char * SCI_METHOD DescribeProperty(const char *name) override {
+		return osBash.DescribeProperty(name);
+	}
+	Sci_Position SCI_METHOD PropertySet(const char *key, const char *val) override;
+	const char * SCI_METHOD DescribeWordListSets() override {
+		return osBash.DescribeWordListSets();
+	}
+	Sci_Position SCI_METHOD WordListSet(int n, const char *wl) override;
+	void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
+	void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
+
+	void * SCI_METHOD PrivateCall(int, void *) override {
+		return 0;
+	}
+
+	int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) override {
+		return subStyles.Allocate(styleBase, numberStyles);
+	}
+	int SCI_METHOD SubStylesStart(int styleBase) override {
+		return subStyles.Start(styleBase);
+	}
+	int SCI_METHOD SubStylesLength(int styleBase) override {
+		return subStyles.Length(styleBase);
+	}
+	int SCI_METHOD StyleFromSubStyle(int subStyle) override {
+		const int styleBase = subStyles.BaseStyle(subStyle);
+		return styleBase;
+	}
+	int SCI_METHOD PrimaryStyleFromStyle(int style) override {
+		return style;
+	}
+	void SCI_METHOD FreeSubStyles() override {
+		subStyles.Free();
+	}
+	void SCI_METHOD SetIdentifiers(int style, const char *identifiers) override {
+		subStyles.SetIdentifiers(style, identifiers);
+	}
+	int SCI_METHOD DistanceToSecondaryStyles() override {
+		return 0;
+	}
+	const char *SCI_METHOD GetSubStyleBases() override {
+		return styleSubable;
+	}
+
+	static ILexer *LexerFactoryBash() {
+		return new LexerBash();
+	}
+};
+
+Sci_Position SCI_METHOD LexerBash::PropertySet(const char *key, const char *val) {
+	if (osBash.PropertySet(&options, key, val)) {
+		return 0;
+	}
+	return -1;
+}
+
+Sci_Position SCI_METHOD LexerBash::WordListSet(int n, const char *wl) {
+	WordList *wordListN = 0;
+	switch (n) {
+	case 0:
+		wordListN = &keywords;
+		break;
+	}
+	Sci_Position firstModification = -1;
+	if (wordListN) {
+		WordList wlNew;
+		wlNew.Set(wl);
+		if (*wordListN != wlNew) {
+			wordListN->Set(wl);
+			firstModification = 0;
+		}
+	}
+	return firstModification;
+}
+
+void SCI_METHOD LexerBash::Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) {
 	WordList cmdDelimiter, bashStruct, bashStruct_in;
 	cmdDelimiter.Set("| || |& & && ; ;; ( ) { }");
 	bashStruct.Set("if elif fi while until else then do done esac eval");
@@ -237,11 +407,15 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in
 	};
 	QuoteStackCls QuoteStack;
 
+	const WordClassifier &classifierIdentifiers = subStyles.Classifier(SCE_SH_IDENTIFIER);
+	const WordClassifier &classifierScalars = subStyles.Classifier(SCE_SH_SCALAR);
+
 	int numBase = 0;
 	int digit;
 	Sci_PositionU endPos = startPos + length;
 	int cmdState = BASH_CMD_START;
 	int testExprType = 0;
+	LexAccessor styler(pAccess);
 
 	// Always backtracks to the start of a line that is not a continuation
 	// of the previous line (i.e. start of a bash command segment)
@@ -306,6 +480,11 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in
 					char s[500];
 					char s2[10];
 					sc.GetCurrent(s, sizeof(s));
+					int identifierStyle = SCE_SH_IDENTIFIER;
+					int subStyle = classifierIdentifiers.ValueFor(s);
+					if (subStyle >= 0) {
+						identifierStyle = subStyle;
+					}
 					// allow keywords ending in a whitespace or command delimiter
 					s2[0] = static_cast<char>(sc.ch);
 					s2[1] = '\0';
@@ -317,7 +496,7 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in
 						else if (strcmp(s, "do") == 0 && keywordEnds)
 							cmdStateNew = BASH_CMD_START;
 						else
-							sc.ChangeState(SCE_SH_IDENTIFIER);
+							sc.ChangeState(identifierStyle);
 						sc.SetState(SCE_SH_DEFAULT);
 						break;
 					}
@@ -327,42 +506,49 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in
 							cmdStateNew = BASH_CMD_TEST;
 							testExprType = 0;
 						} else
-							sc.ChangeState(SCE_SH_IDENTIFIER);
+							sc.ChangeState(identifierStyle);
 					}
 					// detect bash construct keywords
 					else if (bashStruct.InList(s)) {
 						if (cmdState == BASH_CMD_START && keywordEnds)
 							cmdStateNew = BASH_CMD_START;
 						else
-							sc.ChangeState(SCE_SH_IDENTIFIER);
+							sc.ChangeState(identifierStyle);
 					}
 					// 'for'|'case'|'select' needs 'in'|'do' to be highlighted later
 					else if (bashStruct_in.InList(s)) {
 						if (cmdState == BASH_CMD_START && keywordEnds)
 							cmdStateNew = BASH_CMD_WORD;
 						else
-							sc.ChangeState(SCE_SH_IDENTIFIER);
+							sc.ChangeState(identifierStyle);
 					}
 					// disambiguate option items and file test operators
 					else if (s[0] == '-') {
 						if (cmdState != BASH_CMD_TEST)
-							sc.ChangeState(SCE_SH_IDENTIFIER);
+							sc.ChangeState(identifierStyle);
 					}
 					// disambiguate keywords and identifiers
 					else if (cmdState != BASH_CMD_START
 						  || !(keywords.InList(s) && keywordEnds)) {
-						sc.ChangeState(SCE_SH_IDENTIFIER);
+						sc.ChangeState(identifierStyle);
 					}
 					sc.SetState(SCE_SH_DEFAULT);
 				}
 				break;
 			case SCE_SH_IDENTIFIER:
-				if (sc.chPrev == '\\') {	// for escaped chars
-					sc.ForwardSetState(SCE_SH_DEFAULT);
-				} else if (!setWord.Contains(sc.ch)) {
-					sc.SetState(SCE_SH_DEFAULT);
-				} else if (cmdState == BASH_CMD_ARITH && !setWordStart.Contains(sc.ch)) {
-					sc.SetState(SCE_SH_DEFAULT);
+				if (sc.chPrev == '\\' || !setWord.Contains(sc.ch) ||
+					  (cmdState == BASH_CMD_ARITH && !setWordStart.Contains(sc.ch))) {
+					char s[500];
+					sc.GetCurrent(s, sizeof(s));
+					int subStyle = classifierIdentifiers.ValueFor(s);
+					if (subStyle >= 0) {
+						sc.ChangeState(subStyle);
+					}
+					if (sc.chPrev == '\\') {	// for escaped chars
+						sc.ForwardSetState(SCE_SH_DEFAULT);
+					} else {
+						sc.SetState(SCE_SH_DEFAULT);
+					}
 				}
 				break;
 			case SCE_SH_NUMBER:
@@ -516,6 +702,12 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in
 				break;
 			case SCE_SH_SCALAR:	// variable names
 				if (!setParam.Contains(sc.ch)) {
+					char s[500];
+					sc.GetCurrent(s, sizeof(s));
+					int subStyle = classifierScalars.ValueFor(&s[1]); // skip the $
+					if (subStyle >= 0) {
+						sc.ChangeState(subStyle);
+					}
 					if (sc.LengthCurrent() == 1) {
 						// Special variable: $(, $_ etc.
 						sc.ForwardSetState(SCE_SH_DEFAULT);
@@ -799,23 +991,12 @@ static void ColouriseBashDoc(Sci_PositionU startPos, Sci_Position length, int in
 	sc.Complete();
 }
 
-static bool IsCommentLine(Sci_Position line, Accessor &styler) {
-	Sci_Position pos = styler.LineStart(line);
-	Sci_Position eol_pos = styler.LineStart(line + 1) - 1;
-	for (Sci_Position i = pos; i < eol_pos; i++) {
-		char ch = styler[i];
-		if (ch == '#')
-			return true;
-		else if (ch != ' ' && ch != '\t')
-			return false;
-	}
-	return false;
-}
+void SCI_METHOD LexerBash::Fold(Sci_PositionU startPos, Sci_Position length, int, IDocument *pAccess) {
+	if(!options.fold)
+		return;
+
+	LexAccessor styler(pAccess);
 
-static void FoldBashDoc(Sci_PositionU startPos, Sci_Position length, int, WordList *[],
-						Accessor &styler) {
-	bool foldComment = styler.GetPropertyInt("fold.comment") != 0;
-	bool foldCompact = styler.GetPropertyInt("fold.compact", 1) != 0;
 	Sci_PositionU endPos = startPos + length;
 	int visibleChars = 0;
 	int skipHereCh = 0;
@@ -824,6 +1005,8 @@ static void FoldBashDoc(Sci_PositionU startPos, Sci_Position length, int, WordLi
 	int levelCurrent = levelPrev;
 	char chNext = styler[startPos];
 	int styleNext = styler.StyleAt(startPos);
+	char word[8] = { '\0' }; // we're not interested in long words anyway
+	unsigned int wordlen = 0;
 	for (Sci_PositionU i = startPos; i < endPos; i++) {
 		char ch = chNext;
 		chNext = styler.SafeGetCharAt(i + 1);
@@ -831,7 +1014,7 @@ static void FoldBashDoc(Sci_PositionU startPos, Sci_Position length, int, WordLi
 		styleNext = styler.StyleAt(i + 1);
 		bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n');
 		// Comment folding
-		if (foldComment && atEOL && IsCommentLine(lineCurrent, styler))
+		if (options.foldComment && atEOL && IsCommentLine(lineCurrent, styler))
 		{
 			if (!IsCommentLine(lineCurrent - 1, styler)
 				&& IsCommentLine(lineCurrent + 1, styler))
@@ -840,6 +1023,19 @@ static void FoldBashDoc(Sci_PositionU startPos, Sci_Position length, int, WordLi
 					 && !IsCommentLine(lineCurrent + 1, styler))
 				levelCurrent--;
 		}
+		if (style == SCE_SH_WORD) {
+			if ((wordlen + 1) < sizeof(word))
+				word[wordlen++] = ch;
+			if (styleNext != style) {
+				word[wordlen] = '\0';
+				wordlen = 0;
+				if (strcmp(word, "if") == 0 || strcmp(word, "case") == 0 || strcmp(word, "do") == 0) {
+					levelCurrent++;
+				} else if (strcmp(word, "fi") == 0 || strcmp(word, "esac") == 0 || strcmp(word, "done") == 0) {
+					levelCurrent--;
+				}
+			}
+		}
 		if (style == SCE_SH_OPERATOR) {
 			if (ch == '{') {
 				levelCurrent++;
@@ -865,7 +1061,7 @@ static void FoldBashDoc(Sci_PositionU startPos, Sci_Position length, int, WordLi
 		}
 		if (atEOL) {
 			int lev = levelPrev;
-			if (visibleChars == 0 && foldCompact)
+			if (visibleChars == 0 && options.foldCompact)
 				lev |= SC_FOLDLEVELWHITEFLAG;
 			if ((levelCurrent > levelPrev) && (visibleChars > 0))
 				lev |= SC_FOLDLEVELHEADERFLAG;
@@ -884,9 +1080,4 @@ static void FoldBashDoc(Sci_PositionU startPos, Sci_Position length, int, WordLi
 	styler.SetLevel(lineCurrent, levelPrev | flagsNext);
 }
 
-static const char * const bashWordListDesc[] = {
-	"Keywords",
-	0
-};
-
-LexerModule lmBash(SCLEX_BASH, ColouriseBashDoc, "bash", FoldBashDoc, bashWordListDesc);
+LexerModule lmBash(SCLEX_BASH, LexerBash::LexerFactoryBash, "bash", bashWordListDesc);

scintilla/lexers/LexCPP.cxx

@@ -201,17 +201,27 @@ struct EscapeSequence {
 
 std::string GetRestOfLine(LexAccessor &styler, Sci_Position start, bool allowSpace) {
 	std::string restOfLine;
-	Sci_Position i =0;
+	Sci_Position line = styler.GetLine(start);
+	Sci_Position pos = start;
+	Sci_Position endLine = styler.LineEnd(line);
 	char ch = styler.SafeGetCharAt(start, '\n');
-	const Sci_Position endLine = styler.LineEnd(styler.GetLine(start));
-	while (((start+i) < endLine) && (ch != '\r')) {
-		const char chNext = styler.SafeGetCharAt(start + i + 1, '\n');
-		if (ch == '/' && (chNext == '/' || chNext == '*'))
-			break;
-		if (allowSpace || (ch != ' '))
-			restOfLine += ch;
-		i++;
-		ch = chNext;
+	while (pos < endLine) {
+		if (ch == '\\' && ((pos + 1) == endLine)) {
+			// Continuation line
+			line++;
+			pos = styler.LineStart(line);
+			endLine = styler.LineEnd(line);
+			ch = styler.SafeGetCharAt(pos, '\n');
+		} else {
+			const char chNext = styler.SafeGetCharAt(pos + 1, '\n');
+			if (ch == '/' && (chNext == '/' || chNext == '*'))
+				break;
+			if (allowSpace || (ch != ' ')) {
+				restOfLine += ch;
+			}
+			pos++;
+			ch = chNext;
+		}
 	}
 	return restOfLine;
 }
@@ -242,7 +252,11 @@ class LinePPState {
 		return level >= 0 && level < 32;
 	}
 	int maskLevel() const noexcept {
-		return 1 << level;
+		if (level >= 0) {
+			return 1 << level;
+		} else {
+			return 1;
+		}
 	}
 public:
 	LinePPState() : state(0), ifTaken(0), level(-1) {
@@ -470,7 +484,8 @@ class LexerCPP : public ILexerWithMetaData {
 	bool caseSensitive;
 	CharacterSet setWord;
 	CharacterSet setNegationOp;
-	CharacterSet setArithmethicOp;
+	CharacterSet setAddOp;
+	CharacterSet setMultOp;
 	CharacterSet setRelOp;
 	CharacterSet setLogicalOp;
 	CharacterSet setWordStart;
@@ -511,7 +526,8 @@ public:
 		caseSensitive(caseSensitive_),
 		setWord(CharacterSet::setAlphaNum, "._", 0x80, true),
 		setNegationOp(CharacterSet::setNone, "!"),
-		setArithmethicOp(CharacterSet::setNone, "+-/*%"),
+		setAddOp(CharacterSet::setNone, "+-"),
+		setMultOp(CharacterSet::setNone, "*/%"),
 		setRelOp(CharacterSet::setNone, "=!<>"),
 		setLogicalOp(CharacterSet::setNone, "|&"),
 		subStyles(styleSubable, 0x80, 0x40, activeFlag) {
@@ -1275,7 +1291,7 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 							// Ensure only one chosen out of #if .. #elif .. #elif .. #else .. #endif
 							if (!preproc.CurrentIfTaken()) {
 								// Similar to #if
-								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 2, true);
+								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 4, true);
 								const bool ifGood = EvaluateExpression(restOfLine, preprocessorDefinitions);
 								if (ifGood) {
 									preproc.InvertCurrentLevel();
@@ -1614,13 +1630,15 @@ void LexerCPP::EvaluateTokens(std::vector<std::string> &tokens, const SymbolTabl
 	}
 
 	// Evaluate expressions in precedence order
-	enum precedence { precArithmetic, precRelative, precLogical };
-	for (int prec=precArithmetic; prec <= precLogical; prec++) {
+	enum precedence { precMult, precAdd, precRelative
+		, precLogical, /* end marker */ precLast };
+	for (int prec = precMult; prec < precLast; prec++) {
 		// Looking at 3 tokens at a time so end at 2 before end
 		for (size_t k=0; (k+2)<tokens.size();) {
 			const char chOp = tokens[k+1][0];
 			if (
-				((prec==precArithmetic) && setArithmethicOp.Contains(chOp)) ||
+				((prec==precMult) && setMultOp.Contains(chOp)) ||
+				((prec==precAdd) && setAddOp.Contains(chOp)) ||
 				((prec==precRelative) && setRelOp.Contains(chOp)) ||
 				((prec==precLogical) && setLogicalOp.Contains(chOp))
 				) {
@@ -1653,11 +1671,9 @@ void LexerCPP::EvaluateTokens(std::vector<std::string> &tokens, const SymbolTabl
 					result = valA || valB;
 				else if (tokens[k+1] == "&&")
 					result = valA && valB;
-				char sResult[30];
-				sprintf(sResult, "%d", result);
 				std::vector<std::string>::iterator itInsert =
 					tokens.erase(tokens.begin() + k, tokens.begin() + k + 3);
-				tokens.insert(itInsert, sResult);
+				tokens.insert(itInsert, std::to_string(result));
 			} else {
 				k++;
 			}

scintilla/lexers/LexFortran.cxx

@@ -481,11 +481,11 @@ static void FoldFortranDoc(Sci_PositionU startPos, Sci_Position length, int init
 	int levelDeltaNext = 0;
 
 	const unsigned int nComL = 3; // defines how many comment lines should be before they are folded
-	Sci_Position nComColB[nComL];
+	Sci_Position nComColB[nComL] = {};
 	Sci_Position nComColF[nComL] = {};
-	Sci_Position nComCur;
-	bool comLineB[nComL];
-	bool comLineF[nComL];
+	Sci_Position nComCur = 0;
+	bool comLineB[nComL] = {};
+	bool comLineF[nComL] = {};
 	bool comLineCur;
 	Sci_Position nLineTotal = styler.GetLine(styler.Length()-1) + 1;
 	if (foldComment) {

scintilla/lexers/LexMarkdown.cxx

@@ -145,6 +145,7 @@ static void ColorizeMarkdownDoc(Sci_PositionU startPos, Sci_Position length, int
                                WordList **, Accessor &styler) {
     Sci_PositionU endPos = startPos + length;
     int precharCount = 0;
+    bool isLinkNameDetecting = false;
     // Don't advance on a new loop iteration and retry at the same position.
     // Useful in the corner case of having to start at the beginning file position
     // in the default state.
@@ -337,6 +338,27 @@ static void ColorizeMarkdownDoc(Sci_PositionU startPos, Sci_Position length, int
                 ++precharCount;
         }
 
+        // Any link
+        if (sc.state == SCE_MARKDOWN_LINK) {
+            if (sc.Match("](") && sc.GetRelative(-1) != '\\') {
+              sc.Forward(2);
+              isLinkNameDetecting = true;
+            }
+            else if (sc.Match("]:") && sc.GetRelative(-1) != '\\') {
+              sc.Forward(2);
+              sc.SetState(SCE_MARKDOWN_DEFAULT);
+            }
+            else if (!isLinkNameDetecting && sc.ch == ']' && sc.GetRelative(-1) != '\\') {
+              sc.Forward();
+              sc.SetState(SCE_MARKDOWN_DEFAULT);
+            }
+            else if (isLinkNameDetecting && sc.ch == ')' && sc.GetRelative(-1) != '\\') {
+              sc.Forward();
+              sc.SetState(SCE_MARKDOWN_DEFAULT);
+              isLinkNameDetecting = false;
+            }
+        }
+
         // New state anywhere in doc
         if (sc.state == SCE_MARKDOWN_DEFAULT) {
             if (sc.atLineStart && sc.ch == '#') {
@@ -344,38 +366,16 @@ static void ColorizeMarkdownDoc(Sci_PositionU startPos, Sci_Position length, int
                 freezeCursor = true;
             }
             // Links and Images
-            if (sc.Match("![") || sc.ch == '[') {
-                Sci_Position i = 0, j = 0, k = 0;
-                Sci_Position len = endPos - sc.currentPos;
-                while (i < len && (sc.GetRelative(++i) != ']' || sc.GetRelative(i - 1) == '\\'))
-                    ;
-                if (sc.GetRelative(i) == ']') {
-                    j = i;
-                    if (sc.GetRelative(++i) == '(') {
-                        while (i < len && (sc.GetRelative(++i) != ')' || sc.GetRelative(i - 1) == '\\'))
-                            ;
-                        if (sc.GetRelative(i) == ')')
-                            k = i;
-                    }
-                    else if (sc.GetRelative(i) == '[' || sc.GetRelative(++i) == '[') {
-                        while (i < len && (sc.GetRelative(++i) != ']' || sc.GetRelative(i - 1) == '\\'))
-                            ;
-                        if (sc.GetRelative(i) == ']')
-                            k = i;
-                    }
-                }
-                // At least a link text
-                if (j) {
-                    sc.SetState(SCE_MARKDOWN_LINK);
-                    sc.Forward(j);
-                    // Also has a URL or reference portion
-                    if (k)
-                        sc.Forward(k - j);
-                    sc.ForwardSetState(SCE_MARKDOWN_DEFAULT);
-                }
+            if (sc.Match("![")) {
+              sc.SetState(SCE_MARKDOWN_LINK);
+              sc.Forward(2);
+            }
+            else if (sc.ch == '[' && sc.GetRelative(-1) != '\\') {
+              sc.SetState(SCE_MARKDOWN_LINK);
+              sc.Forward();
             }
             // Code - also a special case for alternate inside spacing
-            if (sc.Match("``") && sc.GetRelative(3) != ' ' && AtTermStart(sc)) {
+            else if (sc.Match("``") && sc.GetRelative(3) != ' ' && AtTermStart(sc)) {
                 sc.SetState(SCE_MARKDOWN_CODE2);
                 sc.Forward();
             }

scintilla/lexers/LexVerilog.cxx

@@ -57,7 +57,11 @@ class LinePPState {
 		return level >= 0 && level < 32;
 	}
 	int maskLevel() const {
-		return 1 << level;
+		if (level >= 0) {
+			return 1 << level;
+		} else {
+			return 1;
+		}
 	}
 public:
 	LinePPState() : state(0), ifTaken(0), level(-1) {

scintilla/lexlib/CharacterCategory.cxx

@@ -19,7 +19,7 @@ namespace {
 
 const int catRanges[] = {
 //++Autogenerated -- start of section automatically generated
-// Created with Python 3.6.1,  Unicode 9.0.0
+// Created with Python 3.7.0,  Unicode 11.0.0
 25,
 1046,
 1073,
@@ -651,9 +651,7 @@ const int catRanges[] = {
 43773,
 43811,
 43857,
-44061,
-44065,
-45341,
+44033,
 45361,
 45388,
 45437,
@@ -672,7 +670,7 @@ const int catRanges[] = {
 47389,
 47620,
 48509,
-48644,
+48612,
 48753,
 48829,
 49178,
@@ -730,6 +728,8 @@ const int catRanges[] = {
 65265,
 65347,
 65405,
+65445,
+65491,
 65540,
 66245,
 66371,
@@ -746,11 +746,13 @@ const int catRanges[] = {
 68509,
 68561,
 68605,
+68612,
+68989,
 70660,
 71357,
 71364,
 71645,
-72325,
+72293,
 72794,
 72805,
 73830,
@@ -812,7 +814,10 @@ const int catRanges[] = {
 81546,
 81749,
 81779,
-81821,
+81796,
+81841,
+81861,
+81917,
 81957,
 82022,
 82077,
@@ -849,7 +854,8 @@ const int catRanges[] = {
 85509,
 85572,
 85669,
-85725,
+85713,
+85757,
 86053,
 86118,
 86173,
@@ -886,7 +892,8 @@ const int catRanges[] = {
 89651,
 89693,
 89892,
-89949,
+89925,
+90141,
 90149,
 90182,
 90269,
@@ -969,7 +976,7 @@ const int catRanges[] = {
 98173,
 98309,
 98342,
-98461,
+98437,
 98468,
 98749,
 98756,
@@ -1000,7 +1007,7 @@ const int catRanges[] = {
 102404,
 102437,
 102470,
-102557,
+102545,
 102564,
 102845,
 102852,
@@ -1034,7 +1041,7 @@ const int catRanges[] = {
 106013,
 106020,
 106109,
-106533,
+106501,
 106566,
 106653,
 106660,
@@ -1042,7 +1049,7 @@ const int catRanges[] = {
 106948,
 107069,
 107076,
-108413,
+108389,
 108452,
 108486,
 108581,
@@ -1229,10 +1236,11 @@ const int catRanges[] = {
 137501,
 137632,
 137693,
-137732,
+137729,
 139121,
 139139,
-139172,
+139169,
+139268,
 149821,
 149828,
 149981,
@@ -1337,7 +1345,7 @@ const int catRanges[] = {
 197636,
 198755,
 198788,
-200477,
+200509,
 200708,
 200869,
 200932,
@@ -1463,6 +1471,9 @@ const int catRanges[] = {
 233425,
 233473,
 233789,
+233984,
+235389,
+235424,
 235537,
 235805,
 236037,
@@ -1476,7 +1487,7 @@ const int catRanges[] = {
 237126,
 237189,
 237220,
-237309,
+237286,
 237317,
 237405,
 237569,
@@ -1486,7 +1497,7 @@ const int catRanges[] = {
 241441,
 242531,
 243717,
-245469,
+245597,
 245605,
 245760,
 245793,
@@ -1849,7 +1860,7 @@ const int catRanges[] = {
 266755,
 267197,
 267283,
-268285,
+268317,
 268805,
 269223,
 269349,
@@ -1940,8 +1951,6 @@ const int catRanges[] = {
 292501,
 293778,
 293973,
-294909,
-294933,
 296189,
 296981,
 297341,
@@ -2030,13 +2039,9 @@ const int catRanges[] = {
 356053,
 357085,
 357141,
-358237,
-358325,
 358717,
 358741,
-359005,
-359829,
-359965,
+360445,
 360448,
 361981,
 361985,
@@ -2239,7 +2244,7 @@ const int catRanges[] = {
 378929,
 378957,
 378993,
-379069,
+379389,
 380949,
 381789,
 381813,
@@ -2302,7 +2307,7 @@ const int catRanges[] = {
 401380,
 401437,
 401572,
-402909,
+402973,
 402980,
 406013,
 406037,
@@ -2331,7 +2336,7 @@ const int catRanges[] = {
 636637,
 636949,
 638980,
-1309405,
+1310237,
 1310724,
 1311395,
 1311428,
@@ -2563,12 +2568,14 @@ const int catRanges[] = {
 1373440,
 1373473,
 1373504,
-1373693,
+1373665,
 1373696,
 1373857,
 1373888,
 1373921,
-1373981,
+1373952,
+1373985,
+1374045,
 1375972,
 1376003,
 1376065,
@@ -2606,7 +2613,7 @@ const int catRanges[] = {
 1384292,
 1384337,
 1384356,
-1384413,
+1384421,
 1384456,
 1384772,
 1385669,
@@ -2919,7 +2926,7 @@ const int catRanges[] = {
 2121732,
 2122762,
 2122909,
-2123268,
+2123172,
 2123817,
 2123844,
 2124105,
@@ -3010,12 +3017,12 @@ const int catRanges[] = {
 2179748,
 2179869,
 2179876,
-2180765,
+2180829,
 2180869,
 2180989,
 2181093,
 2181130,
-2181405,
+2181437,
 2181649,
 2181949,
 2182148,
@@ -3054,9 +3061,22 @@ const int catRanges[] = {
 2201601,
 2203261,
 2203466,
-2203677,
+2203652,
+2204805,
+2204957,
+2205192,
+2205533,
 2214922,
 2215933,
+2220036,
+2220970,
+2221284,
+2221341,
+2221572,
+2222277,
+2222634,
+2222769,
+2222941,
 2228230,
 2228261,
 2228294,
@@ -3078,6 +3098,8 @@ const int catRanges[] = {
 2234298,
 2234321,
 2234461,
+2234810,
+2234845,
 2234884,
 2235709,
 2235912,
@@ -3090,7 +3112,9 @@ const int catRanges[] = {
 2238141,
 2238152,
 2238481,
-2238621,
+2238596,
+2238630,
+2238717,
 2238980,
 2240101,
 2240145,
@@ -3104,7 +3128,7 @@ const int catRanges[] = {
 2242534,
 2242596,
 2242737,
-2242885,
+2242853,
 2242993,
 2243037,
 2243080,
@@ -3160,7 +3184,7 @@ const int catRanges[] = {
 2254493,
 2254500,
 2254685,
-2254725,
+2254693,
 2254756,
 2254790,
 2254853,
@@ -3195,7 +3219,8 @@ const int catRanges[] = {
 2263921,
 2263965,
 2263985,
-2264029,
+2264005,
+2264061,
 2265092,
 2266630,
 2266725,
@@ -3249,7 +3274,7 @@ const int catRanges[] = {
 2283528,
 2283869,
 2285572,
-2286429,
+2286461,
 2286501,
 2286598,
 2286661,
@@ -3261,6 +3286,13 @@ const int catRanges[] = {
 2287505,
 2287605,
 2287645,
+2293764,
+2295174,
+2295269,
+2295558,
+2295589,
+2295665,
+2295709,
 2298880,
 2299905,
 2300936,
@@ -3268,6 +3300,30 @@ const int catRanges[] = {
 2301565,
 2301924,
 2301981,
+2310148,
+2310181,
+2310500,
+2311781,
+2311974,
+2312004,
+2312037,
+2312177,
+2312421,
+2312477,
+2312708,
+2312741,
+2312934,
+2312997,
+2313092,
+2314397,
+2314436,
+2314565,
+2314982,
+2315013,
+2315089,
+2315172,
+2315217,
+2315389,
 2316292,
 2318141,
 2326532,
@@ -3297,6 +3353,45 @@ const int catRanges[] = {
 2332294,
 2332325,
 2332413,
+2334724,
+2334973,
+2334980,
+2335069,
+2335076,
+2336293,
+2336509,
+2336581,
+2336637,
+2336645,
+2336733,
+2336741,
+2336964,
+2336997,
+2337053,
+2337288,
+2337629,
+2337796,
+2338013,
+2338020,
+2338109,
+2338116,
+2339142,
+2339325,
+2339333,
+2339421,
+2339430,
+2339493,
+2339526,
+2339557,
+2339588,
+2339645,
+2339848,
+2340189,
+2350084,
+2350693,
+2350758,
+2350833,
+2350909,
 2359300,
 2388829,
 2392073,
@@ -3338,6 +3433,11 @@ const int catRanges[] = {
 2977565,
 2977700,
 2978333,
+3000320,
+3001345,
+3002378,
+3003121,
+3003261,
 3006468,
 3008701,
 3009028,
@@ -3347,13 +3447,15 @@ const int catRanges[] = {
 3011171,
 3011613,
 3013635,
-3013693,
+3013725,
 3014660,
-3210685,
+3210845,
 3211268,
 3235453,
 3538948,
-3539037,
+3548157,
+3550724,
+3563421,
 3637252,
 3640701,
 3640836,
@@ -3388,10 +3490,12 @@ const int catRanges[] = {
 3819589,
 3819701,
 3819741,
+3824650,
+3825309,
 3825685,
 3828477,
 3828746,
-3829341,
+3829565,
 3833856,
 3834689,
 3835520,
@@ -3520,6 +3624,12 @@ const int catRanges[] = {
 4008797,
 4008913,
 4008989,
+4034090,
+4035989,
+4036010,
+4036115,
+4036138,
+4036285,
 4046852,
 4047005,
 4047012,
@@ -3603,8 +3713,6 @@ const int catRanges[] = {
 4071434,
 4071869,
 4071957,
-4072957,
-4072981,
 4074909,
 4075029,
 4076989,
@@ -3616,18 +3724,20 @@ const int catRanges[] = {
 4081981,
 4082197,
 4082269,
+4082709,
+4082909,
 4087829,
 4095860,
 4096021,
-4119165,
+4119229,
 4119573,
 4119997,
 4120085,
-4120317,
+4120413,
 4120597,
 4124317,
 4124693,
-4127421,
+4127549,
 4128789,
 4129181,
 4129301,
@@ -3638,22 +3748,26 @@ const int catRanges[] = {
 4133149,
 4133397,
 4134365,
+4136981,
+4137373,
 4137493,
-4137981,
-4138005,
-4138269,
-4138517,
-4138557,
-4138613,
 4139005,
 4139029,
-4139421,
-4139541,
-4140029,
-4141077,
-4141661,
+4140605,
+4140661,
+4140797,
+4140885,
+4140925,
+4140949,
+4142205,
+4142613,
+4142941,
 4143125,
-4143165,
+4143229,
+4143637,
+4145181,
+4148245,
+4148701,
 4194308,
 5561085,
 5562372,
@@ -3662,6 +3776,8 @@ const int catRanges[] = {
 5702621,
 5702660,
 5887069,
+5887492,
+6126653,
 6225924,
 6243293,
 29360186,

scintilla/lexlib/PropSetSimple.cxx

@@ -117,7 +117,7 @@ static int ExpandAllInPlace(const PropSetSimple &props, std::string &withVars, i
 			innerVarStart = withVars.find("$(", varStart+2);
 		}
 
-		std::string var(withVars.c_str(), varStart + 2, varEnd - varStart - 2);
+		std::string var(withVars, varStart + 2, varEnd - varStart - 2);
 		std::string val = props.Get(var.c_str());
 
 		if (blankVars.contains(var.c_str())) {

scintilla/scintilla_changes.patch

@@ -62,7 +62,7 @@ diff --git scintilla/src/Catalogue.cxx scintilla/src/Catalogue.cxx
 index ed47aa8..e58f1ab 100644
 --- scintilla/src/Catalogue.cxx
 +++ scintilla/src/Catalogue.cxx
-@@ -77,125 +77,50 @@ int Scintilla_LinkLexers() {
+@@ -77,128 +77,50 @@ int Scintilla_LinkLexers() {
  
  //++Autogenerated -- run scripts/LexGen.py to regenerate
  //**\(\tLINK_LEXER(\*);\n\)
@@ -137,6 +137,7 @@ index ed47aa8..e58f1ab 100644
 -	LINK_LEXER(lmModula);
 -	LINK_LEXER(lmMSSQL);
 -	LINK_LEXER(lmMySQL);
+-	LINK_LEXER(lmNim);
 -	LINK_LEXER(lmNimrod);
 -	LINK_LEXER(lmNncrontab);
 +	// We use Octave instead of Matlab
@@ -164,6 +165,7 @@ index ed47aa8..e58f1ab 100644
 -	LINK_LEXER(lmRegistry);
  	LINK_LEXER(lmRuby);
  	LINK_LEXER(lmRust);
+-	LINK_LEXER(lmSAS);
 -	LINK_LEXER(lmScriptol);
 -	LINK_LEXER(lmSmalltalk);
 -	LINK_LEXER(lmSML);
@@ -172,6 +174,7 @@ index ed47aa8..e58f1ab 100644
 -	LINK_LEXER(lmSpice);
  	LINK_LEXER(lmSQL);
 -	LINK_LEXER(lmSrec);
+-	LINK_LEXER(lmStata);
 -	LINK_LEXER(lmSTTXT);
 -	LINK_LEXER(lmTACL);
 -	LINK_LEXER(lmTADS3);

scintilla/src/CaseConvert.cxx

@@ -55,6 +55,7 @@ int symmetricCaseConversionRanges[] = {
 1218,1217,7,2,
 1233,1232,48,2,
 1377,1329,38,1,
+4304,7312,43,1,
 7681,7680,75,2,
 7841,7840,48,2,
 7936,7944,8,1,
@@ -79,6 +80,7 @@ int symmetricCaseConversionRanges[] = {
 66776,66736,36,1,
 68800,68736,51,1,
 71872,71840,32,1,
+93792,93760,32,1,
 125218,125184,34,1,
 
 //--Autogenerated -- end of section automatically generated
@@ -184,6 +186,9 @@ int symmetricCaseConversions[] = {
 1016,1015,
 1019,1018,
 1231,1216,
+4349,7357,
+4350,7358,
+4351,7359,
 7545,42877,
 7549,11363,
 8017,8025,
@@ -233,6 +238,7 @@ int symmetricCaseConversions[] = {
 42899,42898,
 42933,42932,
 42935,42934,
+42937,42936,
 43859,42931,
 
 //--Autogenerated -- end of section automatically generated
@@ -589,7 +595,7 @@ public:
 	}
 	virtual ~CaseConverter() = default;
 	bool Initialised() const {
-		return characters.size() > 0;
+		return !characters.empty();
 	}
 	void Add(int character, const char *conversion) {
 		characterToConversion.emplace_back(character, conversion);
@@ -597,11 +603,11 @@ public:
 	const char *Find(int character) {
 		const std::vector<int>::iterator it = std::lower_bound(characters.begin(), characters.end(), character);
 		if (it == characters.end())
-			return 0;
+			return nullptr;
 		else if (*it == character)
 			return conversions[it - characters.begin()].conversion;
 		else
-			return 0;
+			return nullptr;
 	}
 	size_t CaseConvertString(char *converted, size_t sizeConverted, const char *mixed, size_t lenMixed) override {
 		size_t lenConverted = 0;
@@ -609,7 +615,7 @@ public:
 		unsigned char bytes[UTF8MaxBytes + 1]{};
 		while (mixedPos < lenMixed) {
 			const unsigned char leadByte = mixed[mixedPos];
-			const char *caseConverted = 0;
+			const char *caseConverted = nullptr;
 			size_t lenMixedChar = 1;
 			if (UTF8IsAscii(leadByte)) {
 				caseConverted = Find(leadByte);
@@ -774,7 +780,7 @@ CaseConverter *ConverterForConversion(enum CaseConversion conversion) {
 	case CaseConversionLower:
 		return &caseConvLow;
 	}
-	return 0;
+	return nullptr;
 }
 
 }

scintilla/src/Catalogue.cxx

@@ -33,7 +33,7 @@ const LexerModule *Catalogue::Find(int language) {
 			return lm;
 		}
 	}
-	return 0;
+	return nullptr;
 }
 
 const LexerModule *Catalogue::Find(const char *languageName) {
@@ -45,7 +45,7 @@ const LexerModule *Catalogue::Find(const char *languageName) {
 			}
 		}
 	}
-	return 0;
+	return nullptr;
 }
 
 void Catalogue::AddLexerModule(LexerModule *plm) {

scintilla/src/CellBuffer.cxx

@@ -7,11 +7,13 @@
 
 #include <cstddef>
 #include <cstdlib>
+#include <cassert>
 #include <cstring>
 #include <cstdio>
 #include <cstdarg>
 
 #include <stdexcept>
+#include <string>
 #include <vector>
 #include <algorithm>
 #include <memory>
@@ -27,17 +29,53 @@
 
 namespace Scintilla {
 
+struct CountWidths {
+	// Measures the number of characters in a string divided into those
+	// from the Base Multilingual Plane and those from other planes.
+	Sci::Position countBasePlane;
+	Sci::Position countOtherPlanes;
+	CountWidths(Sci::Position countBasePlane_=0, Sci::Position countOtherPlanes_=0) noexcept :
+		countBasePlane(countBasePlane_),
+		countOtherPlanes(countOtherPlanes_) {
+	}
+	CountWidths operator-() const noexcept {
+		return CountWidths(-countBasePlane , -countOtherPlanes);
+	}
+	Sci::Position WidthUTF32() const noexcept {
+		// All code points take one code unit in UTF-32.
+		return countBasePlane + countOtherPlanes;
+	}
+	Sci::Position WidthUTF16() const noexcept {
+		// UTF-16 takes 2 code units for other planes
+		return countBasePlane + 2 * countOtherPlanes;
+	}
+	void CountChar(int lenChar) noexcept {
+		if (lenChar == 4) {
+			countOtherPlanes++;
+		} else {
+			countBasePlane++;
+		}
+	}
+};
+
 class ILineVector {
 public:
 	virtual void Init() = 0;
 	virtual void SetPerLine(PerLine *pl) = 0;
 	virtual void InsertText(Sci::Line line, Sci::Position delta) = 0;
 	virtual void InsertLine(Sci::Line line, Sci::Position position, bool lineStart) = 0;
-	virtual void SetLineStart(Sci::Line line, Sci::Position position) = 0;
+	virtual void SetLineStart(Sci::Line line, Sci::Position position) noexcept = 0;
 	virtual void RemoveLine(Sci::Line line) = 0;
 	virtual Sci::Line Lines() const noexcept = 0;
 	virtual Sci::Line LineFromPosition(Sci::Position pos) const noexcept = 0;
 	virtual Sci::Position LineStart(Sci::Line line) const noexcept = 0;
+	virtual void InsertCharacters(Sci::Line line, CountWidths delta) = 0;
+	virtual void SetLineCharactersWidth(Sci::Line line, CountWidths width) = 0;
+	virtual int LineCharacterIndex() const noexcept = 0;
+	virtual bool AllocateLineCharacterIndex(int lineCharacterIndex, Sci::Line lines) = 0;
+	virtual bool ReleaseLineCharacterIndex(int lineCharacterIndex) = 0;
+	virtual Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const noexcept = 0;
+	virtual Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const noexcept = 0;
 	virtual ~ILineVector() {}
 };
 
@@ -45,12 +83,61 @@ public:
 
 using namespace Scintilla;
 
+template <typename POS>
+class LineStartIndex {
+public:
+	int refCount;
+	Partitioning<POS> starts;
+
+	LineStartIndex() : refCount(0), starts(4) {
+		// Minimal initial allocation
+	}
+	// Deleted so LineStartIndex objects can not be copied.
+	LineStartIndex(const LineStartIndex &) = delete;
+	LineStartIndex(LineStartIndex &&) = delete;
+	void operator=(const LineStartIndex &) = delete;
+	void operator=(LineStartIndex &&) = delete;
+	virtual ~LineStartIndex() {
+		starts.DeleteAll();
+	}
+	bool Allocate(Sci::Line lines) {
+		refCount++;
+		Sci::Position length = starts.PositionFromPartition(starts.Partitions());
+		for (Sci::Line line = starts.Partitions(); line < lines; line++) {
+			// Produce an ascending sequence that will be filled in with correct widths later
+			length++;
+			starts.InsertPartition(static_cast<POS>(line), static_cast<POS>(length));
+		}
+		return refCount == 1;
+	}
+	bool Release() {
+		if (refCount == 1) {
+			starts.DeleteAll();
+		}
+		refCount--;
+		return refCount == 0;
+	}
+	bool Active() const noexcept {
+		return refCount > 0;
+	}
+	Sci::Position LineWidth(Sci::Line line) const noexcept {
+		return starts.PositionFromPartition(static_cast<POS>(line) + 1) -
+			starts.PositionFromPartition(static_cast<POS>(line));
+	}
+	void SetLineWidth(Sci::Line line, Sci::Position width) {
+		const Sci::Position widthCurrent = LineWidth(line);
+		starts.InsertText(static_cast<POS>(line), static_cast<POS>(width - widthCurrent));
+	}
+};
+
 template <typename POS>
 class LineVector : public ILineVector {
 	Partitioning<POS> starts;
 	PerLine *perLine;
+	LineStartIndex<POS> startsUTF16;
+	LineStartIndex<POS> startsUTF32;
 public:
-	LineVector() : starts(256), perLine(0) {
+	LineVector() : starts(256), perLine(nullptr) {
 		Init();
  	}
 	// Deleted so LineVector objects can not be copied.
@@ -65,7 +152,9 @@ public:
 		if (perLine) {
 			perLine->Init();
 		}
- 	}
+		startsUTF32.starts.DeleteAll();
+		startsUTF16.starts.DeleteAll();
+	}
 	void SetPerLine(PerLine *pl) override {
 		perLine = pl;
 	}
@@ -73,18 +162,33 @@ public:
 		starts.InsertText(static_cast<POS>(line), static_cast<POS>(delta));
 	}
 	void InsertLine(Sci::Line line, Sci::Position position, bool lineStart) override {
-		starts.InsertPartition(static_cast<POS>(line), static_cast<POS>(position));
+		const POS lineAsPos = static_cast<POS>(line);
+		starts.InsertPartition(lineAsPos, static_cast<POS>(position));
+		if (startsUTF32.Active()) {
+			startsUTF32.starts.InsertPartition(lineAsPos,
+				static_cast<POS>(startsUTF32.starts.PositionFromPartition(lineAsPos - 1) + 1));
+		}
+		if (startsUTF16.Active()) {
+			startsUTF16.starts.InsertPartition(lineAsPos,
+				static_cast<POS>(startsUTF16.starts.PositionFromPartition(lineAsPos - 1) + 1));
+		}
 		if (perLine) {
 			if ((line > 0) && lineStart)
 				line--;
 			perLine->InsertLine(line);
 		}
 	}
-	void SetLineStart(Sci::Line line, Sci::Position position) override {
+	void SetLineStart(Sci::Line line, Sci::Position position) noexcept override {
 		starts.SetPartitionStartPosition(static_cast<POS>(line), static_cast<POS>(position));
 	}
 	void RemoveLine(Sci::Line line) override {
 		starts.RemovePartition(static_cast<POS>(line));
+		if (startsUTF32.Active()) {
+			startsUTF32.starts.RemovePartition(static_cast<POS>(line));
+		}
+		if (startsUTF16.Active()) {
+			startsUTF16.starts.RemovePartition(static_cast<POS>(line));
+		}
 		if (perLine) {
 			perLine->RemoveLine(line);
 		}
@@ -98,6 +202,71 @@ public:
 	Sci::Position LineStart(Sci::Line line) const noexcept override {
 		return starts.PositionFromPartition(static_cast<POS>(line));
 	}
+	void InsertCharacters(Sci::Line line, CountWidths delta) override {
+		if (startsUTF32.Active()) {
+			startsUTF32.starts.InsertText(static_cast<POS>(line), static_cast<POS>(delta.WidthUTF32()));
+		}
+		if (startsUTF16.Active()) {
+			startsUTF16.starts.InsertText(static_cast<POS>(line), static_cast<POS>(delta.WidthUTF16()));
+		}
+	}
+	void SetLineCharactersWidth(Sci::Line line, CountWidths width) override {
+		if (startsUTF32.Active()) {
+			assert(startsUTF32.starts.Partitions() == starts.Partitions());
+			startsUTF32.SetLineWidth(line, width.WidthUTF32());
+		}
+		if (startsUTF16.Active()) {
+			assert(startsUTF16.starts.Partitions() == starts.Partitions());
+			startsUTF16.SetLineWidth(line, width.WidthUTF16());
+		}
+	}
+
+	int LineCharacterIndex() const noexcept override {
+		int retVal = 0;
+		if (startsUTF32.Active()) {
+			retVal |= SC_LINECHARACTERINDEX_UTF32;
+		}
+		if (startsUTF16.Active()) {
+			retVal |= SC_LINECHARACTERINDEX_UTF16;
+		}
+		return retVal;
+	}
+	bool AllocateLineCharacterIndex(int lineCharacterIndex, Sci::Line lines) override {
+		bool changed = false;
+		if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF32) != 0) {
+			changed = startsUTF32.Allocate(lines) || changed;
+			assert(startsUTF32.starts.Partitions() == starts.Partitions());
+		}
+		if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF16) != 0) {
+			changed = startsUTF16.Allocate(lines) || changed;
+			assert(startsUTF16.starts.Partitions() == starts.Partitions());
+		}
+		return changed;
+	}
+	bool ReleaseLineCharacterIndex(int lineCharacterIndex) override {
+		bool changed = false;
+		if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF32) != 0) {
+			changed = startsUTF32.Release() || changed;
+		}
+		if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF16) != 0) {
+			changed = startsUTF16.Release() || changed;
+		}
+		return changed;
+	}
+	Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const noexcept override {
+		if (lineCharacterIndex == SC_LINECHARACTERINDEX_UTF32) {
+			return startsUTF32.starts.PositionFromPartition(static_cast<POS>(line));
+		} else {
+			return startsUTF16.starts.PositionFromPartition(static_cast<POS>(line));
+		}
+	}
+	Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const noexcept override {
+		if (lineCharacterIndex == SC_LINECHARACTERINDEX_UTF32) {
+			return static_cast<Sci::Line>(startsUTF32.starts.PartitionFromPosition(static_cast<POS>(pos)));
+		} else {
+			return static_cast<Sci::Line>(startsUTF16.starts.PartitionFromPosition(static_cast<POS>(pos)));
+		}
+	}
 };
 
 Action::Action() {
@@ -363,6 +532,7 @@ void UndoHistory::CompletedRedoStep() {
 CellBuffer::CellBuffer(bool hasStyles_, bool largeDocument_) :
 	hasStyles(hasStyles_), largeDocument(largeDocument_) {
 	readOnly = false;
+	utf8Substance = false;
 	utf8LineEnds = 0;
 	collectingUndo = true;
 	if (largeDocument)
@@ -479,7 +649,7 @@ bool CellBuffer::SetStyleFor(Sci::Position position, Sci::Position lengthStyle,
 const char *CellBuffer::DeleteChars(Sci::Position position, Sci::Position deleteLength, bool &startSequence) {
 	// InsertString and DeleteChars are the bottleneck though which all changes occur
 	PLATFORM_ASSERT(deleteLength > 0);
-	const char *data = 0;
+	const char *data = nullptr;
 	if (!readOnly) {
 		if (collectingUndo) {
 			// Save into the undo/redo stack, but only the characters - not the formatting
@@ -504,10 +674,19 @@ void CellBuffer::Allocate(Sci::Position newSize) {
 	}
 }
 
+void CellBuffer::SetUTF8Substance(bool utf8Substance_) {
+	if (utf8Substance != utf8Substance_) {
+		utf8Substance = utf8Substance_;
+		ResetLineEnds();
+	}
+}
+
 void CellBuffer::SetLineEndTypes(int utf8LineEnds_) {
 	if (utf8LineEnds != utf8LineEnds_) {
+		const int indexes = plv->LineCharacterIndex();
 		utf8LineEnds = utf8LineEnds_;
 		ResetLineEnds();
+		AllocateLineCharacterIndex(indexes);
 	}
 }
 
@@ -534,6 +713,23 @@ void CellBuffer::SetPerLine(PerLine *pl) {
 	plv->SetPerLine(pl);
 }
 
+int CellBuffer::LineCharacterIndex() const noexcept {
+	return plv->LineCharacterIndex();
+}
+
+void CellBuffer::AllocateLineCharacterIndex(int lineCharacterIndex) {
+	if (utf8Substance) {
+		if (plv->AllocateLineCharacterIndex(lineCharacterIndex, Lines())) {
+			// Changed so recalculate whole file
+			RecalculateIndexLineStarts(0, Lines() - 1);
+		}
+	}
+}
+
+void CellBuffer::ReleaseLineCharacterIndex(int lineCharacterIndex) {
+	plv->ReleaseLineCharacterIndex(lineCharacterIndex);
+}
+
 Sci::Line CellBuffer::Lines() const noexcept {
 	return plv->Lines();
 }
@@ -551,6 +747,14 @@ Sci::Line CellBuffer::LineFromPosition(Sci::Position pos) const noexcept {
 	return plv->LineFromPosition(pos);
 }
 
+Sci::Position CellBuffer::IndexLineStart(Sci::Line line, int lineCharacterIndex) const noexcept {
+	return plv->IndexLineStart(line, lineCharacterIndex);
+}
+
+Sci::Line CellBuffer::LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const noexcept {
+	return plv->LineFromPositionIndex(pos, lineCharacterIndex);
+}
+
 bool CellBuffer::IsReadOnly() const {
 	return readOnly;
 }
@@ -611,6 +815,37 @@ bool CellBuffer::UTF8LineEndOverlaps(Sci::Position position) const {
 	return UTF8IsSeparator(bytes) || UTF8IsSeparator(bytes+1) || UTF8IsNEL(bytes+1);
 }
 
+bool CellBuffer::UTF8IsCharacterBoundary(Sci::Position position) const {
+	assert(position >= 0 && position <= Length());
+	if (position > 0) {
+		std::string back;
+		for (int i = 0; i < UTF8MaxBytes; i++) {
+			const Sci::Position posBack = position - i;
+			if (posBack < 0) {
+				return false;
+			}
+			back.insert(0, 1, substance.ValueAt(posBack));
+			if (!UTF8IsTrailByte(back.front())) {
+				if (i > 0) {
+					// Have reached a non-trail
+					const int cla = UTF8Classify(reinterpret_cast<const unsigned char*>(back.data()), back.size());
+					if ((cla & UTF8MaskInvalid) || (cla != i)) {
+						return false;
+					}
+				}
+				break;
+			}
+		}
+	}
+	if (position < Length()) {
+		const unsigned char fore = substance.ValueAt(position);
+		if (UTF8IsTrailByte(fore)) {
+			return false;
+		}
+	}
+	return true;
+}
+
 void CellBuffer::ResetLineEnds() {
 	// Reinitialize line data -- too much work to preserve
 	plv->Init();
@@ -647,6 +882,42 @@ void CellBuffer::ResetLineEnds() {
 	}
 }
 
+namespace {
+
+CountWidths CountCharacterWidthsUTF8(const char *s, size_t len) noexcept {
+	CountWidths cw;
+	size_t remaining = len;
+	while (remaining > 0) {
+		const int utf8Status = UTF8Classify(reinterpret_cast<const unsigned char*>(s), len);
+		const int lenChar = utf8Status & UTF8MaskWidth;
+		cw.CountChar(lenChar);
+		s += lenChar;
+		remaining -= lenChar;
+	}
+	return cw;
+}
+
+}
+
+bool CellBuffer::MaintainingLineCharacterIndex() const noexcept {
+	return plv->LineCharacterIndex() != SC_LINECHARACTERINDEX_NONE;
+}
+
+void CellBuffer::RecalculateIndexLineStarts(Sci::Line lineFirst, Sci::Line lineLast) {
+	std::string text;
+	Sci::Position posLineEnd = LineStart(lineFirst);
+	for (Sci::Line line = lineFirst; line <= lineLast; line++) {
+		// Find line start and end, retrieve text of line, count characters and update line width
+		const Sci::Position posLineStart = posLineEnd;
+		posLineEnd = LineStart(line+1);
+		const Sci::Position width = posLineEnd - posLineStart;
+		text.resize(width);
+		GetCharRange(const_cast<char *>(text.data()), posLineStart, width);
+		const CountWidths cw = CountCharacterWidthsUTF8(text.data(), text.size());
+		plv->SetLineCharactersWidth(line, cw);
+	}
+}
+
 void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::Position insertLength) {
 	if (insertLength == 0)
 		return;
@@ -658,12 +929,27 @@ void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::P
 		breakingUTF8LineEnd = UTF8LineEndOverlaps(position);
 	}
 
+	const Sci::Line linePosition = plv->LineFromPosition(position);
+	Sci::Line lineInsert = linePosition + 1;
+
+	// A simple insertion is one that inserts valid text on a single line at a character boundary
+	bool simpleInsertion = false;
+
+	const bool maintainingIndex = MaintainingLineCharacterIndex();
+
+	// Check for breaking apart a UTF-8 sequence and inserting invalid UTF-8
+	if (utf8Substance && maintainingIndex) {
+		// Actually, don't need to check that whole insertion is valid just that there
+		// are no potential fragments at ends.
+		simpleInsertion = UTF8IsCharacterBoundary(position) &&
+			UTF8IsValid(s, insertLength);
+	}
+
 	substance.InsertFromArray(position, s, 0, insertLength);
 	if (hasStyles) {
 		style.InsertValue(position, insertLength, 0);
 	}
 
-	Sci::Line lineInsert = plv->LineFromPosition(position) + 1;
 	const bool atLineStart = plv->LineStart(lineInsert-1) == position;
 	// Point all the lines after the insertion point further along in the buffer
 	plv->InsertText(lineInsert-1, insertLength);
@@ -683,6 +969,7 @@ void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::P
 		if (ch == '\r') {
 			InsertLine(lineInsert, (position + i) + 1, atLineStart);
 			lineInsert++;
+			simpleInsertion = false;
 		} else if (ch == '\n') {
 			if (chPrev == '\r') {
 				// Patch up what was end of line
@@ -691,11 +978,13 @@ void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::P
 				InsertLine(lineInsert, (position + i) + 1, atLineStart);
 				lineInsert++;
 			}
+			simpleInsertion = false;
 		} else if (utf8LineEnds) {
 			const unsigned char back3[3] = {chBeforePrev, chPrev, ch};
 			if (UTF8IsSeparator(back3) || UTF8IsNEL(back3+1)) {
 				InsertLine(lineInsert, (position + i) + 1, atLineStart);
 				lineInsert++;
+				simpleInsertion = false;
 			}
 		}
 		chBeforePrev = chPrev;
@@ -706,6 +995,7 @@ void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::P
 		if (ch == '\r') {
 			// End of line already in buffer so drop the newly created one
 			RemoveLine(lineInsert - 1);
+			simpleInsertion = false;
 		}
 	} else if (utf8LineEnds && !UTF8IsAscii(chAfter)) {
 		// May have end of UTF-8 line end in buffer and start in insertion
@@ -715,21 +1005,33 @@ void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::P
 			if (UTF8IsSeparator(back3)) {
 				InsertLine(lineInsert, (position + insertLength + j) + 1, atLineStart);
 				lineInsert++;
+				simpleInsertion = false;
 			}
 			if ((j == 0) && UTF8IsNEL(back3+1)) {
 				InsertLine(lineInsert, (position + insertLength + j) + 1, atLineStart);
 				lineInsert++;
+				simpleInsertion = false;
 			}
 			chBeforePrev = chPrev;
 			chPrev = chAt;
 		}
 	}
+	if (maintainingIndex) {
+		if (simpleInsertion) {
+			const CountWidths cw = CountCharacterWidthsUTF8(s, insertLength);
+			plv->InsertCharacters(linePosition, cw);
+		} else {
+			RecalculateIndexLineStarts(linePosition, lineInsert - 1);
+		}
+	}
 }
 
 void CellBuffer::BasicDeleteChars(Sci::Position position, Sci::Position deleteLength) {
 	if (deleteLength == 0)
 		return;
 
+	Sci::Line lineRecalculateStart = INVALID_POSITION;
+
 	if ((position == 0) && (deleteLength == substance.Length())) {
 		// If whole buffer is being deleted, faster to reinitialise lines data
 		// than to delete each line.
@@ -738,11 +1040,37 @@ void CellBuffer::BasicDeleteChars(Sci::Position position, Sci::Position deleteLe
 		// Have to fix up line positions before doing deletion as looking at text in buffer
 		// to work out which lines have been removed
 
-		Sci::Line lineRemove = plv->LineFromPosition(position) + 1;
+		const Sci::Line linePosition = plv->LineFromPosition(position);
+		Sci::Line lineRemove = linePosition + 1;
+
 		plv->InsertText(lineRemove-1, - (deleteLength));
 		const unsigned char chPrev = substance.ValueAt(position - 1);
 		const unsigned char chBefore = chPrev;
 		unsigned char chNext = substance.ValueAt(position);
+
+		// Check for breaking apart a UTF-8 sequence
+		// Needs further checks that text is UTF-8 or that some other break apart is occurring
+		if (utf8Substance && MaintainingLineCharacterIndex()) {
+			const Sci::Position posEnd = position + deleteLength;
+			const Sci::Line lineEndRemove = plv->LineFromPosition(posEnd);
+			const bool simpleDeletion =
+				(linePosition == lineEndRemove) &&
+				UTF8IsCharacterBoundary(position) && UTF8IsCharacterBoundary(posEnd);
+			if (simpleDeletion) {
+				std::string text(deleteLength, '\0');
+				GetCharRange(const_cast<char *>(text.data()), position, deleteLength);
+				if (UTF8IsValid(text.data(), text.size())) {
+					// Everything is good
+					const CountWidths cw = CountCharacterWidthsUTF8(text.data(), text.size());
+					plv->InsertCharacters(linePosition, -cw);
+				} else {
+					lineRecalculateStart = linePosition;
+				}
+			} else {
+				lineRecalculateStart = linePosition;
+			}
+		}
+
 		bool ignoreNL = false;
 		if (chPrev == '\r' && chNext == '\n') {
 			// Move back one
@@ -791,6 +1119,9 @@ void CellBuffer::BasicDeleteChars(Sci::Position position, Sci::Position deleteLe
 		}
 	}
 	substance.DeleteRange(position, deleteLength);
+	if (lineRecalculateStart >= 0) {
+		RecalculateIndexLineStarts(lineRecalculateStart, lineRecalculateStart);
+	}
 	if (hasStyles) {
 		style.DeleteRange(position, deleteLength);
 	}
@@ -816,7 +1147,7 @@ void CellBuffer::EndUndoAction() {
 
 void CellBuffer::AddUndoAction(Sci::Position token, bool mayCoalesce) {
 	bool startSequence;
-	uh.AppendAction(containerAction, token, 0, 0, startSequence, mayCoalesce);
+	uh.AppendAction(containerAction, token, nullptr, 0, startSequence, mayCoalesce);
 }
 
 void CellBuffer::DeleteUndoHistory() {

scintilla/src/CellBuffer.h

@@ -113,6 +113,7 @@ private:
 	SplitVector<char> substance;
 	SplitVector<char> style;
 	bool readOnly;
+	bool utf8Substance;
 	int utf8LineEnds;
 
 	bool collectingUndo;
@@ -121,7 +122,10 @@ private:
 	std::unique_ptr<ILineVector> plv;
 
 	bool UTF8LineEndOverlaps(Sci::Position position) const;
+	bool UTF8IsCharacterBoundary(Sci::Position position) const;
 	void ResetLineEnds();
+	void RecalculateIndexLineStarts(Sci::Line lineFirst, Sci::Line lineLast);
+	bool MaintainingLineCharacterIndex() const noexcept;
 	/// Actions without undo
 	void BasicInsertString(Sci::Position position, const char *s, Sci::Position insertLength);
 	void BasicDeleteChars(Sci::Position position, Sci::Position deleteLength);
@@ -148,13 +152,19 @@ public:
 
 	Sci::Position Length() const noexcept;
 	void Allocate(Sci::Position newSize);
+	void SetUTF8Substance(bool utf8Substance_);
 	int GetLineEndTypes() const { return utf8LineEnds; }
 	void SetLineEndTypes(int utf8LineEnds_);
 	bool ContainsLineEnd(const char *s, Sci::Position length) const;
 	void SetPerLine(PerLine *pl);
+	int LineCharacterIndex() const noexcept;
+	void AllocateLineCharacterIndex(int lineCharacterIndex);
+	void ReleaseLineCharacterIndex(int lineCharacterIndex);
 	Sci::Line Lines() const noexcept;
 	Sci::Position LineStart(Sci::Line line) const noexcept;
+	Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const noexcept;
 	Sci::Line LineFromPosition(Sci::Position pos) const noexcept;
+	Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const noexcept;
 	void InsertLine(Sci::Line line, Sci::Position position, bool lineStart);
 	void RemoveLine(Sci::Line line);
 	const char *InsertString(Sci::Position position, const char *s, Sci::Position insertLength, bool &startSequence);

scintilla/src/ContractionState.cxx

@@ -210,7 +210,7 @@ Sci::Line ContractionState<LINE>::DocFromDisplay(Sci::Line lineDisplay) const {
 
 template <typename LINE>
 void ContractionState<LINE>::InsertLines(Sci::Line lineDoc, Sci::Line lineCount) {
-	for (int l = 0; l < lineCount; l++) {
+	for (Sci::Line l = 0; l < lineCount; l++) {
 		InsertLine(lineDoc + l);
 	}
 	Check();

scintilla/src/Document.cxx

@@ -87,8 +87,31 @@ int LexInterface::LineEndTypesSupported() {
 	return 0;
 }
 
+ActionDuration::ActionDuration(double duration_, double minDuration_, double maxDuration_) noexcept :
+	duration(duration_), minDuration(minDuration_), maxDuration(maxDuration_) {
+}
+
+void ActionDuration::AddSample(size_t numberActions, double durationOfActions) noexcept {
+	// Only adjust for multiple actions to avoid instability
+	if (numberActions < 8)
+		return;
+
+	// Alpha value for exponential smoothing.
+	// Most recent value contributes 25% to smoothed value.
+	const double alpha = 0.25;
+
+	const double durationOne = durationOfActions / numberActions;
+	duration = Sci::clamp(alpha * durationOne + (1.0 - alpha) * duration,
+		minDuration, maxDuration);
+}
+
+double ActionDuration::Duration() const noexcept {
+	return duration;
+}
+
 Document::Document(int options) :
-	cb((options & SC_DOCUMENTOPTION_STYLES_NONE) == 0, (options & SC_DOCUMENTOPTION_TEXT_LARGE) != 0) {
+	cb((options & SC_DOCUMENTOPTION_STYLES_NONE) == 0, (options & SC_DOCUMENTOPTION_TEXT_LARGE) != 0),
+	durationStyleOneLine(0.00001, 0.000001, 0.0001) {
 	refCount = 0;
 #ifdef _WIN32
 	eolMode = SC_EOL_CRLF;
@@ -109,7 +132,6 @@ Document::Document(int options) :
 	useTabs = true;
 	tabIndents = true;
 	backspaceUnindents = false;
-	durationStyleOneLine = 0.00001;
 
 	matchesValid = false;
 
@@ -122,6 +144,7 @@ Document::Document(int options) :
 	decorations = DecorationListCreate(IsLarge());
 
 	cb.SetPerLine(this);
+	cb.SetUTF8Substance(SC_CP_UTF8 == dbcsCodePage);
 }
 
 Document::~Document() {
@@ -197,6 +220,8 @@ bool Document::SetDBCSCodePage(int dbcsCodePage_) {
 		dbcsCodePage = dbcsCodePage_;
 		SetCaseFolder(nullptr);
 		cb.SetLineEndTypes(lineEndBitSet & LineEndTypesSupported());
+		cb.SetUTF8Substance(SC_CP_UTF8 == dbcsCodePage);
+		ModifiedAt(0);	// Need to restyle whole document
 		return true;
 	} else {
 		return false;
@@ -296,11 +321,11 @@ Sci::Line Document::MarkerNext(Sci::Line lineStart, int mask) const {
 int Document::AddMark(Sci::Line line, int markerNum) {
 	if (line >= 0 && line <= LinesTotal()) {
 		const int prev = Markers()->AddMark(line, markerNum, LinesTotal());
-		const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, 0, line);
+		const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, nullptr, line);
 		NotifyModified(mh);
 		return prev;
 	} else {
-		return 0;
+		return -1;
 	}
 }
 
@@ -313,19 +338,19 @@ void Document::AddMarkSet(Sci::Line line, int valueSet) {
 		if (m & 1)
 			Markers()->AddMark(line, i, LinesTotal());
 	}
-	const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, 0, line);
+	const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, nullptr, line);
 	NotifyModified(mh);
 }
 
 void Document::DeleteMark(Sci::Line line, int markerNum) {
 	Markers()->DeleteMark(line, markerNum, false);
-	const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, 0, line);
+	const DocModification mh(SC_MOD_CHANGEMARKER, LineStart(line), 0, 0, nullptr, line);
 	NotifyModified(mh);
 }
 
 void Document::DeleteMarkFromHandle(int markerHandle) {
 	Markers()->DeleteMarkFromHandle(markerHandle);
-	DocModification mh(SC_MOD_CHANGEMARKER, 0, 0, 0, 0);
+	DocModification mh(SC_MOD_CHANGEMARKER);
 	mh.line = -1;
 	NotifyModified(mh);
 }
@@ -337,7 +362,7 @@ void Document::DeleteAllMarks(int markerNum) {
 			someChanges = true;
 	}
 	if (someChanges) {
-		DocModification mh(SC_MOD_CHANGEMARKER, 0, 0, 0, 0);
+		DocModification mh(SC_MOD_CHANGEMARKER);
 		mh.line = -1;
 		NotifyModified(mh);
 	}
@@ -423,11 +448,19 @@ Sci::Position Document::VCHomePosition(Sci::Position position) const {
 		return startText;
 }
 
+Sci::Position Document::IndexLineStart(Sci::Line line, int lineCharacterIndex) const {
+	return cb.IndexLineStart(line, lineCharacterIndex);
+}
+
+Sci::Line Document::LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const {
+	return cb.LineFromPositionIndex(pos, lineCharacterIndex);
+}
+
 int SCI_METHOD Document::SetLevel(Sci_Position line, int level) {
 	const int prev = Levels()->SetLevel(static_cast<Sci::Line>(line), level, LinesTotal());
 	if (prev != level) {
 		DocModification mh(SC_MOD_CHANGEFOLD | SC_MOD_CHANGEMARKER,
-		                   LineStart(line), 0, 0, 0, static_cast<Sci::Line>(line));
+		                   LineStart(line), 0, 0, nullptr, static_cast<Sci::Line>(line));
 		mh.foldLevelNow = level;
 		mh.foldLevelPrev = prev;
 		NotifyModified(mh);
@@ -897,8 +930,8 @@ Sci::Position Document::GetRelativePositionUTF16(Sci::Position positionStart, Sc
 int SCI_METHOD Document::GetCharacterAndWidth(Sci_Position position, Sci_Position *pWidth) const {
 	int character;
 	int bytesInCharacter = 1;
+	const unsigned char leadByte = cb.UCharAt(position);
 	if (dbcsCodePage) {
-		const unsigned char leadByte = cb.UCharAt(position);
 		if (SC_CP_UTF8 == dbcsCodePage) {
 			if (UTF8IsAscii(leadByte)) {
 				// Single byte character or invalid
@@ -926,7 +959,7 @@ int SCI_METHOD Document::GetCharacterAndWidth(Sci_Position position, Sci_Positio
 			}
 		}
 	} else {
-		character = cb.CharAt(position);
+		character = leadByte;
 	}
 	if (pWidth) {
 		*pWidth = bytesInCharacter;
@@ -2105,7 +2138,19 @@ const char *Document::SubstituteByPosition(const char *text, Sci::Position *leng
 	if (regex)
 		return regex->SubstituteByPosition(this, text, length);
 	else
-		return 0;
+		return nullptr;
+}
+
+int Document::LineCharacterIndex() const {
+	return cb.LineCharacterIndex();
+}
+
+void Document::AllocateLineCharacterIndex(int lineCharacterIndex) {
+	return cb.AllocateLineCharacterIndex(lineCharacterIndex);
+}
+
+void Document::ReleaseLineCharacterIndex(int lineCharacterIndex) {
+	return cb.ReleaseLineCharacterIndex(lineCharacterIndex);
 }
 
 Sci::Line Document::LinesTotal() const noexcept {
@@ -2191,30 +2236,11 @@ void Document::EnsureStyledTo(Sci::Position pos) {
 }
 
 void Document::StyleToAdjustingLineDuration(Sci::Position pos) {
-	// Place bounds on the duration used to avoid glitches spiking it
-	// and so causing slow styling or non-responsive scrolling
-	const double minDurationOneLine = 0.000001;
-	const double maxDurationOneLine = 0.0001;
-
-	// Alpha value for exponential smoothing.
-	// Most recent value contributes 25% to smoothed value.
-	const double alpha = 0.25;
-
 	const Sci::Line lineFirst = SciLineFromPosition(GetEndStyled());
 	ElapsedPeriod epStyling;
 	EnsureStyledTo(pos);
-	const double durationStyling = epStyling.Duration();
 	const Sci::Line lineLast = SciLineFromPosition(GetEndStyled());
-	if (lineLast >= lineFirst + 8) {
-		// Only adjust for styling multiple lines to avoid instability
-		const double durationOneLine = durationStyling / (lineLast - lineFirst);
-		durationStyleOneLine = alpha * durationOneLine + (1.0 - alpha) * durationStyleOneLine;
-		if (durationStyleOneLine < minDurationOneLine) {
-			durationStyleOneLine = minDurationOneLine;
-		} else if (durationStyleOneLine > maxDurationOneLine) {
-			durationStyleOneLine = maxDurationOneLine;
-		}
-	}
+	durationStyleOneLine.AddSample(lineLast - lineFirst, epStyling.Duration());
 }
 
 void Document::LexerChanged() {
@@ -2235,7 +2261,7 @@ void Document::SetLexInterface(LexInterface *pLexInterface) {
 int SCI_METHOD Document::SetLineState(Sci_Position line, int state) {
 	const int statePrevious = States()->SetLineState(static_cast<Sci::Line>(line), state);
 	if (state != statePrevious) {
-		const DocModification mh(SC_MOD_CHANGELINESTATE, LineStart(line), 0, 0, 0,
+		const DocModification mh(SC_MOD_CHANGELINESTATE, LineStart(line), 0, 0, nullptr,
 			static_cast<Sci::Line>(line));
 		NotifyModified(mh);
 	}
@@ -2284,7 +2310,7 @@ void Document::MarginSetStyles(Sci::Line line, const unsigned char *styles) {
 void Document::MarginClearAll() {
 	const Sci::Line maxEditorLine = LinesTotal();
 	for (Sci::Line l=0; l<maxEditorLine; l++)
-		MarginSetText(l, 0);
+		MarginSetText(l, nullptr);
 	// Free remaining data
 	Margins()->ClearAll();
 }
@@ -2327,7 +2353,7 @@ int Document::AnnotationLines(Sci::Line line) const {
 void Document::AnnotationClearAll() {
 	const Sci::Line maxEditorLine = LinesTotal();
 	for (Sci::Line l=0; l<maxEditorLine; l++)
-		AnnotationSetText(l, 0);
+		AnnotationSetText(l, nullptr);
 	// Free remaining data
 	Annotations()->ClearAll();
 }
@@ -2960,7 +2986,6 @@ std::regex_constants::match_flag_type MatchFlags(const Document *doc, Sci::Posit
 
 template<typename Iterator, typename Regex>
 bool MatchOnLines(const Document *doc, const Regex &regexp, const RESearchRange &resr, RESearch &search) {
-	bool matched = false;
 	std::match_results<Iterator> match;
 
 	// MSVC and libc++ have problems with ^ and $ matching line ends inside a range.
@@ -2974,9 +2999,10 @@ bool MatchOnLines(const Document *doc, const Regex &regexp, const RESearchRange
 	Iterator itStart(doc, resr.startPos);
 	Iterator itEnd(doc, resr.endPos);
 	const std::regex_constants::match_flag_type flagsMatch = MatchFlags(doc, resr.startPos, resr.endPos);
-	matched = std::regex_search(itStart, itEnd, match, regexp, flagsMatch);
+	const bool matched = std::regex_search(itStart, itEnd, match, regexp, flagsMatch);
 #else
 	// Line by line.
+	bool matched = false;
 	for (Sci::Line line = resr.lineRangeStart; line != resr.lineRangeBreak; line += resr.increment) {
 		const Range lineRange = resr.LineRange(line);
 		Iterator itStart(doc, lineRange.start);
@@ -3167,7 +3193,7 @@ const char *BuiltinRegex::SubstituteByPosition(Document *doc, const char *text,
 	substituted.clear();
 	const DocumentIndexer di(doc, doc->Length());
 	search.GrabMatches(di);
-	for (int j = 0; j < *length; j++) {
+	for (Sci::Position j = 0; j < *length; j++) {
 		if (text[j] == '\\') {
 			if (text[j + 1] >= '0' && text[j + 1] <= '9') {
 				const unsigned int patNum = text[j + 1] - '0';

scintilla/src/Document.h

@@ -189,6 +189,26 @@ struct RegexError : public std::runtime_error {
 	RegexError() : std::runtime_error("regex failure") {}
 };
 
+/**
+ * The ActionDuration class stores the average time taken for some action such as styling or
+ * wrapping a line. It is used to decide how many repetitions of that action can be performed
+ * on idle to maximize efficiency without affecting application responsiveness.
+ * The duration changes if the time for the action changes. For example, if a simple lexer is
+ * changed to a complex lexer. Changes are damped and clamped to avoid short periods of easy
+ * or difficult processing moving the value too far leading to inefficiency or poor user
+ * experience.
+ */
+
+class ActionDuration {
+	double duration;
+	const double minDuration;
+	const double maxDuration;
+public:
+	ActionDuration(double duration_, double minDuration_, double maxDuration_) noexcept;
+	void AddSample(size_t numberActions, double durationOfActions) noexcept;
+	double Duration() const noexcept;
+};
+
 /**
  */
 class Document : PerLine, public IDocumentWithLineEnd, public ILoader {
@@ -259,7 +279,7 @@ public:
 	bool useTabs;
 	bool tabIndents;
 	bool backspaceUnindents;
-	double durationStyleOneLine;
+	ActionDuration durationStyleOneLine;
 
 	std::unique_ptr<IDecorationList> decorations;
 
@@ -389,6 +409,8 @@ public:
 	bool IsLineEndPosition(Sci::Position position) const;
 	bool IsPositionInLineEnd(Sci::Position position) const;
 	Sci::Position VCHomePosition(Sci::Position position) const;
+	Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const;
+	Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const;
 
 	int SCI_METHOD SetLevel(Sci_Position line, int level) override;
 	int SCI_METHOD GetLevel(Sci_Position line) const override;
@@ -414,6 +436,9 @@ public:
 	void SetCaseFolder(CaseFolder *pcf_);
 	Sci::Position FindText(Sci::Position minPos, Sci::Position maxPos, const char *search, int flags, Sci::Position *length);
 	const char *SubstituteByPosition(const char *text, Sci::Position *length);
+	int LineCharacterIndex() const;
+	void AllocateLineCharacterIndex(int lineCharacterIndex);
+	void ReleaseLineCharacterIndex(int lineCharacterIndex);
 	Sci::Line LinesTotal() const noexcept;
 
 	void SetDefaultCharClasses(bool includeWordClass);

scintilla/src/EditModel.cxx

@@ -73,5 +73,5 @@ EditModel::EditModel() : braces{} {
 
 EditModel::~EditModel() {
 	pdoc->Release();
-	pdoc = 0;
+	pdoc = nullptr;
 }

scintilla/src/EditView.cxx

@@ -911,7 +911,7 @@ void EditView::DrawEOL(Surface *surface, const EditModel &model, const ViewStyle
 	int alpha = SC_ALPHA_NOALPHA;
 	if (!hideSelection) {
 		const Sci::Position posAfterLineEnd = model.pdoc->LineStart(line + 1);
-		eolInSelection = (lastSubLine == true) ? model.sel.InSelectionForEOL(posAfterLineEnd) : 0;
+		eolInSelection = lastSubLine ? model.sel.InSelectionForEOL(posAfterLineEnd) : 0;
 		alpha = (eolInSelection == 1) ? vsDraw.selAlpha : vsDraw.selAdditionalAlpha;
 	}
 

scintilla/src/Editor.cxx

@@ -21,6 +21,7 @@
 #include <algorithm>
 #include <iterator>
 #include <memory>
+#include <chrono>
 
 #include "Platform.h"
 
@@ -54,6 +55,7 @@
 #include "MarginView.h"
 #include "EditView.h"
 #include "Editor.h"
+#include "ElapsedPeriod.h"
 
 using namespace Scintilla;
 
@@ -89,7 +91,7 @@ static bool IsLastStep(const DocModification &mh) {
 }
 
 Timer::Timer() :
-		ticking(false), ticksToWait(0), tickerID(0) {}
+		ticking(false), ticksToWait(0), tickerID{} {}
 
 Idler::Idler() :
 		state(false), idlerID(0) {}
@@ -103,7 +105,7 @@ static inline bool IsAllSpacesOrTabs(const char *s, unsigned int len) {
 	return true;
 }
 
-Editor::Editor() {
+Editor::Editor() : durationWrapOneLine(0.00001, 0.000001, 0.0001) {
 	ctrlID = 0;
 
 	stylesValid = false;
@@ -181,6 +183,7 @@ Editor::Editor() {
 	needIdleStyling = false;
 
 	modEventMask = SC_MODEVENTMASKALL;
+	commandEvents = true;
 
 	pdoc->AddWatcher(this, 0);
 
@@ -206,7 +209,7 @@ void Editor::SetRepresentations() {
 	reprs.Clear();
 
 	// C0 control set
-	const char * const reps[] = {
+	const char *const reps[] = {
 		"NUL", "SOH", "STX", "ETX", "EOT", "ENQ", "ACK", "BEL",
 		"BS", "HT", "LF", "VT", "FF", "CR", "SO", "SI",
 		"DLE", "DC1", "DC2", "DC3", "DC4", "NAK", "SYN", "ETB",
@@ -220,7 +223,7 @@ void Editor::SetRepresentations() {
 	// C1 control set
 	// As well as Unicode mode, ISO-8859-1 should use these
 	if (IsUnicodeMode()) {
-		const char * const repsC1[] = {
+		const char *const repsC1[] = {
 			"PAD", "HOP", "BPH", "NBH", "IND", "NEL", "SSA", "ESA",
 			"HTS", "HTJ", "VTS", "PLD", "PLU", "RI", "SS2", "SS3",
 			"DCS", "PU1", "PU2", "STS", "CCH", "MW", "SPA", "EPA",
@@ -294,7 +297,7 @@ void Editor::RefreshStyleData() {
 }
 
 Point Editor::GetVisibleOriginInMain() const {
-	return Point(0,0);
+	return Point(0, 0);
 }
 
 PointDocument Editor::DocumentPointFromView(Point ptView) const {
@@ -767,6 +770,7 @@ void Editor::MultipleSelectAdd(AddNumber addNumber) {
 					selectedText.c_str(), searchFlags, &lengthFound);
 				if (pos >= 0) {
 					sel.AddSelection(SelectionRange(pos + lengthFound, pos));
+					ContainerNeedsUpdate(SC_UPDATE_SELECTION);
 					ScrollRange(sel.RangeMain());
 					Redraw();
 					if (addNumber == addOne)
@@ -1532,7 +1536,12 @@ bool Editor::WrapLines(WrapScope ws) {
 				return false;
 			}
 		} else if (ws == WrapScope::wsIdle) {
-			lineToWrapEnd = lineToWrap + LinesOnScreen() + 100;
+			// Try to keep time taken by wrapping reasonable so interaction remains smooth.
+			const double secondsAllowed = 0.01;
+			const Sci::Line linesInAllowedTime = Sci::clamp(
+				static_cast<Sci::Line>(secondsAllowed / durationWrapOneLine.Duration()),
+				LinesOnScreen() + 50, static_cast<Sci::Line>(0x10000));
+			lineToWrapEnd = lineToWrap + linesInAllowedTime;
 		}
 		const Sci::Line lineEndNeedWrap = std::min(wrapPending.end, pdoc->LinesTotal());
 		lineToWrapEnd = std::min(lineToWrapEnd, lineEndNeedWrap);
@@ -1551,6 +1560,8 @@ bool Editor::WrapLines(WrapScope ws) {
 			if (surface) {
 //Platform::DebugPrintf("Wraplines: scope=%0d need=%0d..%0d perform=%0d..%0d\n", ws, wrapPending.start, wrapPending.end, lineToWrap, lineToWrapEnd);
 
+				const Sci::Line linesBeingWrapped = lineToWrapEnd - lineToWrap;
+				ElapsedPeriod epWrapping;
 				while (lineToWrap < lineToWrapEnd) {
 					if (WrapOneLine(surface, lineToWrap)) {
 						wrapOccurred = true;
@@ -1558,6 +1569,7 @@ bool Editor::WrapLines(WrapScope ws) {
 					wrapPending.Wrapped(lineToWrap);
 					lineToWrap++;
 				}
+				durationWrapOneLine.AddSample(linesBeingWrapped, epWrapping.Duration());
 
 				goodTopLine = pcs->DisplayFromDoc(lineDocTop) + std::min(
 					subLineTop, static_cast<Sci::Line>(pcs->GetHeight(lineDocTop)-1));
@@ -2674,9 +2686,11 @@ void Editor::NotifyModified(Document *, DocModification mh, void *) {
 
 	// If client wants to see this modification
 	if (mh.modificationType & modEventMask) {
-		if ((mh.modificationType & (SC_MOD_CHANGESTYLE | SC_MOD_CHANGEINDICATOR)) == 0) {
-			// Real modification made to text of document.
-			NotifyChange();	// Send EN_CHANGE
+		if (commandEvents) {
+			if ((mh.modificationType & (SC_MOD_CHANGESTYLE | SC_MOD_CHANGEINDICATOR)) == 0) {
+				// Real modification made to text of document.
+				NotifyChange();	// Send EN_CHANGE
+			}
 		}
 
 		SCNotification scn = {};
@@ -4155,9 +4169,7 @@ std::string Editor::RangeText(Sci::Position start, Sci::Position end) const {
 	if (start < end) {
 		const Sci::Position len = end - start;
 		std::string ret(len, '\0');
-		for (int i = 0; i < len; i++) {
-			ret[i] = pdoc->CharAt(start + i);
-		}
+		pdoc->GetCharRange(const_cast<char *>(ret.data()), start, len);
 		return ret;
 	}
 	return std::string();
@@ -4455,7 +4467,7 @@ void Editor::DwellEnd(bool mouseMoved) {
 void Editor::MouseLeave() {
 	SetHotSpotRange(nullptr);
 	if (!HaveMouseCapture()) {
-		ptMouseLast = Point(-1,-1);
+		ptMouseLast = Point(-1, -1);
 		DwellEnd(true);
 	}
 }
@@ -4810,7 +4822,7 @@ void Editor::ButtonMoveWithModifiers(Point pt, unsigned int, int modifiers) {
 		if (hotspot.Valid() && !PointIsHotspot(pt))
 			SetHotSpotRange(nullptr);
 
-		if (hotSpotClickPos != INVALID_POSITION && PositionFromLocation(pt,true,true) != hotSpotClickPos) {
+		if (hotSpotClickPos != INVALID_POSITION && PositionFromLocation(pt, true, true) != hotSpotClickPos) {
 			if (inDragDrop == ddNone) {
 				DisplayCursor(Window::cursorText);
 			}
@@ -5051,7 +5063,7 @@ Sci::Position Editor::PositionAfterMaxStyling(Sci::Position posMax, bool scrolli
 	// When scrolling, allow less time to ensure responsive
 	const double secondsAllowed = scrolling ? 0.005 : 0.02;
 
-	const Sci::Line linesToStyle = Sci::clamp(static_cast<int>(secondsAllowed / pdoc->durationStyleOneLine),
+	const Sci::Line linesToStyle = Sci::clamp(static_cast<int>(secondsAllowed / pdoc->durationStyleOneLine.Duration()),
 		10, 0x10000);
 	const Sci::Line stylingMaxLine = std::min(
 		pdoc->SciLineFromPosition(pdoc->GetEndStyled()) + linesToStyle,
@@ -5201,7 +5213,7 @@ void Editor::SetDocPointer(Document *document) {
 	//Platform::DebugPrintf("** %x setdoc to %x\n", pdoc, document);
 	pdoc->RemoveWatcher(this, 0);
 	pdoc->Release();
-	if (document == NULL) {
+	if (!document) {
 		pdoc = new Document(SC_DOCUMENTOPTION_DEFAULT);
 	} else {
 		pdoc = document;
@@ -5520,7 +5532,7 @@ void Editor::NeedShown(Sci::Position pos, Sci::Position len) {
 }
 
 Sci::Position Editor::GetTag(char *tagValue, int tagNumber) {
-	const char *text = 0;
+	const char *text = nullptr;
 	Sci::Position length = 0;
 	if ((tagNumber >= 1) && (tagNumber <= 9)) {
 		char name[3] = "\\?";
@@ -5763,7 +5775,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 			if (wParam == 0)
 				return 0;
 			char *ptr = CharPtrFromSPtr(lParam);
-			unsigned int iChar = 0;
+			size_t iChar = 0;
 			for (; iChar < wParam - 1; iChar++)
 				ptr[iChar] = pdoc->CharAt(iChar);
 			ptr[iChar] = '\0';
@@ -5897,7 +5909,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 				return selectedText.LengthWithTerminator();
 			} else {
 				char *ptr = CharPtrFromSPtr(lParam);
-				unsigned int iChar = 0;
+				size_t iChar = 0;
 				if (selectedText.Length()) {
 					for (; iChar < selectedText.LengthWithTerminator(); iChar++)
 						ptr[iChar] = selectedText.Data()[iChar];
@@ -6017,6 +6029,11 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 			static_cast<Sci::Position>(wParam), lParam),
 			static_cast<Sci::Position>(0), pdoc->Length());
 
+	case SCI_POSITIONRELATIVECODEUNITS:
+		return Sci::clamp(pdoc->GetRelativePositionUTF16(
+			static_cast<Sci::Position>(wParam), lParam),
+			static_cast<Sci::Position>(0), pdoc->Length());
+
 	case SCI_LINESCROLL:
 		ScrollTo(topLine + static_cast<Sci::Line>(lParam));
 		HorizontalScrollTo(xOffset + static_cast<int>(wParam) * static_cast<int>(vs.spaceWidth));
@@ -6491,15 +6508,15 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 
 	case SCI_CLEARTABSTOPS:
 		if (view.ClearTabstops(static_cast<Sci::Line>(wParam))) {
-			const DocModification mh(SC_MOD_CHANGETABSTOPS, 0, 0, 0, 0, static_cast<Sci::Line>(wParam));
-			NotifyModified(pdoc, mh, NULL);
+			const DocModification mh(SC_MOD_CHANGETABSTOPS, 0, 0, 0, nullptr, static_cast<Sci::Line>(wParam));
+			NotifyModified(pdoc, mh, nullptr);
 		}
 		break;
 
 	case SCI_ADDTABSTOP:
 		if (view.AddTabstop(static_cast<Sci::Line>(wParam), static_cast<int>(lParam))) {
-			const DocModification mh(SC_MOD_CHANGETABSTOPS, 0, 0, 0, 0, static_cast<Sci::Line>(wParam));
-			NotifyModified(pdoc, mh, NULL);
+			const DocModification mh(SC_MOD_CHANGETABSTOPS, 0, 0, 0, nullptr, static_cast<Sci::Line>(wParam));
+			NotifyModified(pdoc, mh, nullptr);
 		}
 		break;
 
@@ -6773,6 +6790,23 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 	case SCI_GETIMEINTERACTION:
 		return imeInteraction;
 
+	case SCI_GETLINECHARACTERINDEX:
+		return pdoc->LineCharacterIndex();
+
+	case SCI_ALLOCATELINECHARACTERINDEX:
+		pdoc->AllocateLineCharacterIndex(static_cast<int>(wParam));
+		break;
+
+	case SCI_RELEASELINECHARACTERINDEX:
+		pdoc->ReleaseLineCharacterIndex(static_cast<int>(wParam));
+		break;
+
+	case SCI_LINEFROMINDEXPOSITION:
+		return pdoc->LineFromPositionIndex(static_cast<Sci::Position>(wParam), static_cast<int>(lParam));
+
+	case SCI_INDEXPOSITIONFROMLINE:
+		return pdoc->IndexLineStart(static_cast<Sci::Line>(wParam), static_cast<int>(lParam));
+
 		// Marker definition and setting
 	case SCI_MARKERDEFINE:
 		if (wParam <= MARKER_MAX) {
@@ -7384,7 +7418,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 		return pdoc->decorations->AllOnFor(static_cast<Sci::Position>(wParam));
 
 	case SCI_INDICATORVALUEAT:
-		return pdoc->decorations->ValueAt(static_cast<int>(wParam), static_cast<Sci::Position>(lParam));
+		return pdoc->decorations->ValueAt(static_cast<int>(wParam), lParam);
 
 	case SCI_INDICATORSTART:
 		return pdoc->decorations->Start(static_cast<int>(wParam), lParam);
@@ -7522,11 +7556,15 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 		InvalidateStyleRedraw();
 		break;
 
-	case SCI_SETZOOM:
-		vs.zoomLevel = static_cast<int>(wParam);
-		InvalidateStyleRedraw();
-		NotifyZoom();
-		break;
+	case SCI_SETZOOM: {
+			const int zoomLevel = static_cast<int>(wParam);
+			if (zoomLevel != vs.zoomLevel) {
+				vs.zoomLevel = zoomLevel;
+				InvalidateStyleRedraw();
+				NotifyZoom();
+			}
+			break;
+		}
 
 	case SCI_GETZOOM:
 		return vs.zoomLevel;
@@ -7615,6 +7653,13 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 	case SCI_GETMODEVENTMASK:
 		return modEventMask;
 
+	case SCI_SETCOMMANDEVENTS:
+		commandEvents = static_cast<bool>(wParam);
+		return 0;
+
+	case SCI_GETCOMMANDEVENTS:
+		return commandEvents;
+
 	case SCI_CONVERTEOLS:
 		pdoc->ConvertLineEnds(static_cast<int>(wParam));
 		SetSelection(sel.MainCaret(), sel.MainAnchor());	// Ensure selection inside document
@@ -8179,6 +8224,9 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 	case SCI_COUNTCHARACTERS:
 		return pdoc->CountCharacters(static_cast<Sci::Position>(wParam), lParam);
 
+	case SCI_COUNTCODEUNITS:
+		return pdoc->CountUTF16(static_cast<Sci::Position>(wParam), lParam);
+
 	default:
 		return DefWndProc(iMessage, wParam, lParam);
 	}

scintilla/src/Editor.h

@@ -229,6 +229,7 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 	bool needIdleStyling;
 
 	int modEventMask;
+	bool commandEvents;
 
 	SelectionText drag;
 
@@ -249,6 +250,7 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 
 	// Wrapping support
 	WrapPending wrapPending;
+	ActionDuration durationWrapOneLine;
 
 	bool convertPastes;
 

scintilla/src/ExternalLexer.cxx

@@ -64,7 +64,9 @@ LexerLibrary::LexerLibrary(const char *moduleName_) {
 				// Assign a buffer for the lexer name.
 				char lexname[100] = "";
 				GetLexerName(i, lexname, sizeof(lexname));
-				ExternalLexerModule *lex = new ExternalLexerModule(SCLEX_AUTOMATIC, NULL, lexname, NULL);
+				ExternalLexerModule *lex = new ExternalLexerModule(SCLEX_AUTOMATIC, nullptr, lexname, nullptr);
+				// This is storing a second reference to lex in the Catalogue as well as in modules.
+				// TODO: Should use std::shared_ptr or similar to ensure allocation safety.
 				Catalogue::AddLexerModule(lex);
 
 				// Remember ExternalLexerModule so we don't leak it

scintilla/src/Indicator.cxx

@@ -48,8 +48,7 @@ void Indicator::Draw(Surface *surface, const PRectangle &rc, const PRectangle &r
 		surface->MoveTo(x, irc.top + y);
 		while (x < xLast) {
 			if ((x + 2) > xLast) {
-				if (xLast > x)
-					y = 1;
+				y = 1;
 				x = xLast;
 			} else {
 				x += 2;
@@ -61,7 +60,7 @@ void Indicator::Draw(Surface *surface, const PRectangle &rc, const PRectangle &r
 		const PRectangle rcSquiggle = PixelGridAlign(rc);
 
 		const int width = std::min(4000, static_cast<int>(rcSquiggle.Width()));
-		RGBAImage image(width, 3, 1.0, 0);
+		RGBAImage image(width, 3, 1.0, nullptr);
 		enum { alphaFull = 0xff, alphaSide = 0x2f, alphaSide2=0x5f };
 		for (int x = 0; x < width; x++) {
 			if (x%2) {
@@ -166,7 +165,7 @@ void Indicator::Draw(Surface *surface, const PRectangle &rc, const PRectangle &r
 		IntegerRectangle ircBox(rcBox);
 		// Cap width at 4000 to avoid large allocations when mistakes made
 		const int width = std::min(ircBox.Width(), 4000);
-		RGBAImage image(width, ircBox.Height(), 1.0, 0);
+		RGBAImage image(width, ircBox.Height(), 1.0, nullptr);
 		// Draw horizontal lines top and bottom
 		for (int x=0; x<width; x++) {
 			for (int y = 0; y<ircBox.Height(); y += ircBox.Height() - 1) {

scintilla/src/KeyMap.cxx

@@ -45,6 +45,10 @@ unsigned int KeyMap::Find(int key, int modifiers) const {
 	return (it == kmap.end()) ? 0 : it->second;
 }
 
+const std::map<KeyModifiers, unsigned int> &KeyMap::GetKeyMap() const {
+	return kmap;
+}
+
 #if PLAT_GTK_MACOSX
 #define OS_X_KEYS 1
 #else

scintilla/src/KeyMap.h

@@ -56,6 +56,7 @@ public:
 	void Clear();
 	void AssignCmdKey(int key, int modifiers, unsigned int msg);
 	unsigned int Find(int key, int modifiers) const;	// 0 returned on failure
+	const std::map<KeyModifiers, unsigned int> &GetKeyMap() const;
 };
 
 }

scintilla/src/PerLine.cxx

@@ -349,14 +349,14 @@ const char *LineAnnotation::Text(Sci::Line line) const {
 	if (annotations.Length() && (line >= 0) && (line < annotations.Length()) && annotations[line])
 		return annotations[line].get()+sizeof(AnnotationHeader);
 	else
-		return 0;
+		return nullptr;
 }
 
 const unsigned char *LineAnnotation::Styles(Sci::Line line) const {
 	if (annotations.Length() && (line >= 0) && (line < annotations.Length()) && annotations[line] && MultipleStyles(line))
 		return reinterpret_cast<unsigned char *>(annotations[line].get() + sizeof(AnnotationHeader) + Length(line));
 	else
-		return 0;
+		return nullptr;
 }
 
 static char *AllocateAnnotation(int length, int style) {

scintilla/src/PositionCache.cxx

@@ -386,13 +386,14 @@ SpecialRepresentations::SpecialRepresentations() {
 }
 
 void SpecialRepresentations::SetRepresentation(const char *charBytes, const char *value) {
-	MapRepresentation::iterator it = mapReprs.find(KeyFromString(charBytes, UTF8MaxBytes));
+	const unsigned int key = KeyFromString(charBytes, UTF8MaxBytes);
+	MapRepresentation::iterator it = mapReprs.find(key);
 	if (it == mapReprs.end()) {
 		// New entry so increment for first byte
 		const unsigned char ucStart = charBytes[0];
 		startByteHasReprs[ucStart]++;
 	}
-	mapReprs[KeyFromString(charBytes, UTF8MaxBytes)] = Representation(value);
+	mapReprs[key] = Representation(value);
 }
 
 void SpecialRepresentations::ClearRepresentation(const char *charBytes) {
@@ -574,7 +575,7 @@ PositionCacheEntry::PositionCacheEntry(const PositionCacheEntry &other) :
 }
 
 void PositionCacheEntry::Set(unsigned int styleNumber_, const char *s_,
-	unsigned int len_, XYPOSITION *positions_, unsigned int clock_) {
+	unsigned int len_, const XYPOSITION *positions_, unsigned int clock_) {
 	Clear();
 	styleNumber = styleNumber_;
 	len = len_;

scintilla/src/PositionCache.h

@@ -150,7 +150,7 @@ public:
 	void operator=(const PositionCacheEntry &) = delete;
 	void operator=(PositionCacheEntry &&) = delete;
 	~PositionCacheEntry();
-	void Set(unsigned int styleNumber_, const char *s_, unsigned int len_, XYPOSITION *positions_, unsigned int clock_);
+	void Set(unsigned int styleNumber_, const char *s_, unsigned int len_, const XYPOSITION *positions_, unsigned int clock_);
 	void Clear();
 	bool Retrieve(unsigned int styleNumber_, const char *s_, unsigned int len_, XYPOSITION *positions_) const;
 	static unsigned int Hash(unsigned int styleNumber_, const char *s, unsigned int len_);

scintilla/src/RESearch.cxx

@@ -445,7 +445,7 @@ const char *RESearch::Compile(const char *pattern, Sci::Position length, bool ca
 
 	if (!pattern || !length) {
 		if (sta)
-			return 0;
+			return nullptr;
 		else
 			return badpat("No previous regular expression");
 	}
@@ -727,7 +727,7 @@ const char *RESearch::Compile(const char *pattern, Sci::Position length, bool ca
 		return badpat((posix ? "Unmatched (" : "Unmatched \\("));
 	*mp = END;
 	sta = OKP;
-	return 0;
+	return nullptr;
 }
 
 /*

scintilla/src/RunStyles.cxx

@@ -249,7 +249,7 @@ DISTANCE RunStyles<DISTANCE, STYLE>::Runs() const noexcept {
 
 template <typename DISTANCE, typename STYLE>
 bool RunStyles<DISTANCE, STYLE>::AllSame() const noexcept {
-	for (int run = 1; run < starts->Partitions(); run++) {
+	for (DISTANCE run = 1; run < starts->Partitions(); run++) {
 		if (styles->ValueAt(run) != styles->ValueAt(run - 1))
 			return false;
 	}
@@ -299,7 +299,7 @@ void RunStyles<DISTANCE, STYLE>::Check() const {
 	if (styles->ValueAt(styles->Length()-1) != 0) {
 		throw std::runtime_error("RunStyles: Unused style at end changed.");
 	}
-	for (int j=1; j<styles->Length()-1; j++) {
+	for (ptrdiff_t j=1; j<styles->Length()-1; j++) {
 		if (styles->ValueAt(j) == styles->ValueAt(j-1)) {
 			throw std::runtime_error("RunStyles: Style of a partition same as previous.");
 		}
@@ -308,7 +308,7 @@ void RunStyles<DISTANCE, STYLE>::Check() const {
 
 template class Scintilla::RunStyles<int, int>;
 template class Scintilla::RunStyles<int, char>;
-#if PTRDIFF_MAX != INT_MAX
+#if (PTRDIFF_MAX != INT_MAX) || PLAT_HAIKU
 template class Scintilla::RunStyles<ptrdiff_t, int>;
 template class Scintilla::RunStyles<ptrdiff_t, char>;
 #endif

scintilla/src/ScintillaBase.cxx

@@ -442,12 +442,12 @@ int ScintillaBase::AutoCompleteGetCurrentText(char *buffer) const {
 		const int item = ac.GetSelection();
 		if (item != -1) {
 			const std::string selected = ac.GetValue(item);
-			if (buffer != NULL)
+			if (buffer)
 				memcpy(buffer, selected.c_str(), selected.length()+1);
 			return static_cast<int>(selected.length());
 		}
 	}
-	if (buffer != NULL)
+	if (buffer)
 		*buffer = '\0';
 	return 0;
 }
@@ -592,7 +592,7 @@ public:
 }
 
 LexState::LexState(Document *pdoc_) : LexInterface(pdoc_) {
-	lexCurrent = 0;
+	lexCurrent = nullptr;
 	performingStyle = false;
 	interfaceVersion = lvOriginal;
 	lexLanguage = SCLEX_CONTAINER;
@@ -601,7 +601,7 @@ LexState::LexState(Document *pdoc_) : LexInterface(pdoc_) {
 LexState::~LexState() {
 	if (instance) {
 		instance->Release();
-		instance = 0;
+		instance = nullptr;
 	}
 }
 
@@ -616,7 +616,7 @@ void LexState::SetLexerModule(const LexerModule *lex) {
 	if (lex != lexCurrent) {
 		if (instance) {
 			instance->Release();
-			instance = 0;
+			instance = nullptr;
 		}
 		interfaceVersion = lvOriginal;
 		lexCurrent = lex;
@@ -631,7 +631,7 @@ void LexState::SetLexerModule(const LexerModule *lex) {
 void LexState::SetLexer(uptr_t wParam) {
 	lexLanguage = static_cast<int>(wParam);
 	if (lexLanguage == SCLEX_CONTAINER) {
-		SetLexerModule(0);
+		SetLexerModule(nullptr);
 	} else {
 		const LexerModule *lex = Catalogue::Find(lexLanguage);
 		if (!lex)
@@ -653,7 +653,7 @@ const char *LexState::DescribeWordListSets() {
 	if (instance) {
 		return instance->DescribeWordListSets();
 	} else {
-		return 0;
+		return nullptr;
 	}
 }
 
@@ -674,7 +674,7 @@ void *LexState::PrivateCall(int operation, void *pointer) {
 	if (pdoc && instance) {
 		return instance->PrivateCall(operation, pointer);
 	} else {
-		return 0;
+		return nullptr;
 	}
 }
 
@@ -682,7 +682,7 @@ const char *LexState::PropertyNames() {
 	if (instance) {
 		return instance->PropertyNames();
 	} else {
-		return 0;
+		return nullptr;
 	}
 }
 
@@ -698,7 +698,7 @@ const char *LexState::DescribeProperty(const char *name) {
 	if (instance) {
 		return instance->DescribeProperty(name);
 	} else {
-		return 0;
+		return nullptr;
 	}
 }
 
@@ -805,7 +805,7 @@ const char *LexState::NameOfStyle(int style) {
 	if (instance && (interfaceVersion >= lvMetaData)) {
 		return static_cast<ILexerWithMetaData *>(instance)->NameOfStyle(style);
 	} else {
-		return 0;
+		return nullptr;
 	}
 }
 
@@ -813,7 +813,7 @@ const char *LexState::TagsOfStyle(int style) {
 	if (instance && (interfaceVersion >= lvMetaData)) {
 		return static_cast<ILexerWithMetaData *>(instance)->TagsOfStyle(style);
 	} else {
-		return 0;
+		return nullptr;
 	}
 }
 
@@ -821,7 +821,7 @@ const char *LexState::DescriptionOfStyle(int style) {
 	if (instance && (interfaceVersion >= lvMetaData)) {
 		return static_cast<ILexerWithMetaData *>(instance)->DescriptionOfStyle(style);
 	} else {
-		return 0;
+		return nullptr;
 	}
 }
 

scintilla/src/Style.cxx

@@ -76,13 +76,13 @@ void FontMeasurements::ClearMeasurements() {
 
 Style::Style() : FontSpecification() {
 	Clear(ColourDesired(0, 0, 0), ColourDesired(0xff, 0xff, 0xff),
-	      Platform::DefaultFontSize() * SC_FONT_SIZE_MULTIPLIER, 0, SC_CHARSET_DEFAULT,
+	      Platform::DefaultFontSize() * SC_FONT_SIZE_MULTIPLIER, nullptr, SC_CHARSET_DEFAULT,
 	      SC_WEIGHT_NORMAL, false, false, false, caseMixed, true, true, false);
 }
 
 Style::Style(const Style &source) : FontSpecification(), FontMeasurements() {
 	Clear(ColourDesired(0, 0, 0), ColourDesired(0xff, 0xff, 0xff),
-	      0, 0, 0,
+	      0, nullptr, 0,
 	      SC_WEIGHT_NORMAL, false, false, false, caseMixed, true, true, false);
 	fore = source.fore;
 	back = source.back;
@@ -106,7 +106,7 @@ Style &Style::operator=(const Style &source) {
 	if (this == &source)
 		return * this;
 	Clear(ColourDesired(0, 0, 0), ColourDesired(0xff, 0xff, 0xff),
-	      0, 0, SC_CHARSET_DEFAULT,
+	      0, nullptr, SC_CHARSET_DEFAULT,
 	      SC_WEIGHT_NORMAL, false, false, false, caseMixed, true, true, false);
 	fore = source.fore;
 	back = source.back;

scintilla/src/UniConversion.cxx

@@ -327,6 +327,22 @@ int UTF8DrawBytes(const unsigned char *us, int len) noexcept {
 	return (utf8StatusNext & UTF8MaskInvalid) ? 1 : (utf8StatusNext & UTF8MaskWidth);
 }
 
+bool UTF8IsValid(const char *s, size_t len) noexcept {
+	const unsigned char *us = reinterpret_cast<const unsigned char *>(s);
+	size_t remaining = len;
+	while (remaining > 0) {
+		const int utf8Status = UTF8Classify(us, remaining);
+		if (utf8Status & UTF8MaskInvalid) {
+			return false;
+		} else {
+			const int lenChar = utf8Status & UTF8MaskWidth;
+			us += lenChar;
+			remaining -= lenChar;
+		}
+	}
+	return remaining == 0;
+}
+
 // Replace invalid bytes in UTF-8 with the replacement character
 std::string FixInvalidUTF8(const std::string &text) {
 	std::string result;

scintilla/src/UniConversion.h

@@ -21,6 +21,7 @@ size_t UTF16Length(const char *s, size_t len);
 size_t UTF16FromUTF8(const char *s, size_t len, wchar_t *tbuf, size_t tlen);
 size_t UTF32FromUTF8(const char *s, size_t len, unsigned int *tbuf, size_t tlen);
 unsigned int UTF16FromUTF32Character(unsigned int val, wchar_t *tbuf) noexcept;
+bool UTF8IsValid(const char *s, size_t len) noexcept;
 std::string FixInvalidUTF8(const std::string &text);
 
 extern const unsigned char UTF8BytesOfLead[256];

scintilla/src/ViewStyle.cxx

@@ -403,7 +403,7 @@ void ViewStyle::ResetDefaultStyle() {
 
 void ViewStyle::ClearStyles() {
 	// Reset all styles to be like the default style
-	for (unsigned int i=0; i<styles.size(); i++) {
+	for (size_t i=0; i<styles.size(); i++) {
 		if (i != STYLE_DEFAULT) {
 			styles[i].ClearTo(styles[STYLE_DEFAULT]);
 		}
@@ -429,7 +429,7 @@ int ViewStyle::ExternalMarginWidth() const {
 
 int ViewStyle::MarginFromLocation(Point pt) const {
 	int margin = -1;
-	int x = textStart - fixedColumnWidth;
+	int x = marginInside ? 0 : -fixedColumnWidth;
 	for (size_t i = 0; i < ms.size(); i++) {
 		if ((pt.x >= x) && (pt.x < x + ms[i].width))
 			margin = static_cast<int>(i);
@@ -600,7 +600,7 @@ FontRealised *ViewStyle::Find(const FontSpecification &fs) {
 		// Should always reach here since map was just set for all styles
 		return it->second.get();
 	}
-	return 0;
+	return nullptr;
 }
 
 void ViewStyle::FindMaxAscentDescent() {

scintilla/version.txt

@@ -1 +1 @@
-3100
+3102