Merge remote-tracking branch 'vim/master'

runtime/doc/change.txt

@@ -1688,6 +1688,10 @@ B	When joining lines, don't insert a space between two multi-byte
 	characters.  Overruled by the 'M' flag.
 1	Don't break a line after a one-letter word.  It's broken before it
 	instead (if possible).
+]	Respect textwidth rigorously. With this flag set, no line can be
+	longer than textwidth, unless line-break-prohibition rules make this
+	impossible.  Mainly for CJK scripts and works only if 'encoding' is
+	"utf-8".
 j	Where it makes sense, remove a comment leader when joining lines.  For
 	example, joining:
 		int i;   // the index ~

runtime/doc/eval.txt

@@ -1,4 +1,4 @@
-*eval.txt*	For Vim version 8.2.  Last change: 2020 May 25
+*eval.txt*	For Vim version 8.2.  Last change: 2020 Jun 03
 
 
 		  VIM REFERENCE MANUAL	  by Bram Moolenaar
@@ -2358,7 +2358,6 @@ assert_notmatch({pat}, {text} [, {msg}])
 				Number	assert {pat} not matches {text}
 assert_report({msg})		Number	report a test failure
 assert_true({actual} [, {msg}])	Number	assert {actual} is true
-asin({expr})			Float	arc sine of {expr}
 atan({expr})			Float	arc tangent of {expr}
 atan2({expr1}, {expr2})		Float	arc tangent of {expr1} / {expr2}
 balloon_gettext()		String	current text in the balloon
@@ -2449,12 +2448,12 @@ executable({expr})		Number	1 if executable {expr} exists
 execute({command})		String	execute {command} and get the output
 exepath({expr})			String	full path of the command {expr}
 exists({expr})			Number	|TRUE| if {expr} exists
-extend({expr1}, {expr2} [, {expr3}])
-				List/Dict insert items of {expr2} into {expr1}
 exp({expr})			Float	exponential of {expr}
 expand({expr} [, {nosuf} [, {list}]])
 				any	expand special keywords in {expr}
 expandcmd({expr})		String	expand {expr} like with `:edit`
+extend({expr1}, {expr2} [, {expr3}])
+				List/Dict insert items of {expr2} into {expr1}
 feedkeys({string} [, {mode}])	Number	add key sequence to typeahead buffer
 filereadable({file})		Number	|TRUE| if {file} is a readable file
 filewritable({file})		Number	|TRUE| if {file} is a writable file
@@ -2549,8 +2548,8 @@ histadd({history}, {item})	Number	add an item to a history
 histdel({history} [, {item}])	Number	remove an item from a history
 histget({history} [, {index}])	String	get the item {index} from a history
 histnr({history})		Number	highest index of a history
-hlexists({name})		Number	|TRUE| if highlight group {name} exists
 hlID({name})			Number	syntax ID of highlight group {name}
+hlexists({name})		Number	|TRUE| if highlight group {name} exists
 hostname()			String	name of the machine Vim is running on
 iconv({expr}, {from}, {to})	String	convert encoding of {expr}
 indent({lnum})			Number	indent of line {lnum}
@@ -2650,16 +2649,16 @@ popup_findpreview()		Number	get window ID of preview popup window
 popup_getoptions({id})		Dict	get options of popup window {id}
 popup_getpos({id})		Dict	get position of popup window {id}
 popup_hide({id})		none	hide popup menu {id}
-popup_list()			List	get a list of window IDs of al popups
+popup_list()			List	get a list of window IDs of all popups
 popup_locate({row}, {col})	Number 	get window ID of popup at position
 popup_menu({what}, {options})	Number	create a popup window used as a menu
 popup_move({id}, {options})	none	set position of popup window {id}
 popup_notification({what}, {options})
 				Number	create a notification popup window
-popup_show({id})		none	unhide popup window {id}
 popup_setoptions({id}, {options})
 				none	set options for popup window {id}
 popup_settext({id}, {text})	none	set the text of popup window {id}
+popup_show({id})		none	unhide popup window {id}
 pow({x}, {y})			Float	{x} to the power of {y}
 prevnonblank({lnum})		Number	line nr of non-blank line <= {lnum}
 printf({fmt}, {expr1}...)	String	format text
@@ -2684,8 +2683,8 @@ prop_type_get([{name} [, {props}]])
 prop_type_list([{props}])	List	get list of property types
 pum_getpos()			Dict	position and size of pum if visible
 pumvisible()			Number	whether popup menu is visible
-pyeval({expr})			any	evaluate |Python| expression
 py3eval({expr})			any	evaluate |python3| expression
+pyeval({expr})			any	evaluate |Python| expression
 pyxeval({expr})			any	evaluate |python_x| expression
 rand([{expr}])			Number	get pseudo-random number
 range({expr} [, {max} [, {stride}]])
@@ -2812,9 +2811,9 @@ str2list({expr} [, {utf8}])	List	convert each character of {expr} to
 					ASCII/UTF8 value
 str2nr({expr} [, {base} [, {quoted}]])
 				Number	convert String to Number
-strchars({expr} [, {skipcc}])	Number	character length of the String {expr}
 strcharpart({str}, {start} [, {len}])
 				String	{len} characters of {str} at {start}
+strchars({expr} [, {skipcc}])	Number	character length of the String {expr}
 strdisplaywidth({expr} [, {col}]) Number display length of the String {expr}
 strftime({format} [, {time}])	String	format time with a specified format
 strgetchar({str}, {index})	Number	get char {index} from {str}
@@ -2847,8 +2846,8 @@ systemlist({expr} [, {input}])	List	output of shell command/filter {expr}
 tabpagebuflist([{arg}])		List	list of buffer numbers in tab page
 tabpagenr([{arg}])		Number	number of current or last tab page
 tabpagewinnr({tabarg} [, {arg}]) Number	number of current window in tab page
-taglist({expr} [, {filename}])	List	list of tags matching {expr}
 tagfiles()			List	tags files used
+taglist({expr} [, {filename}])	List	list of tags matching {expr}
 tan({expr})			Float	tangent of {expr}
 tanh({expr})			Float	hyperbolic tangent of {expr}
 tempname()			String	name for a temporary file
@@ -2872,9 +2871,9 @@ term_gettty({buf}, [{input}])	String	get the tty name of a terminal
 term_list()			List	get the list of terminal buffers
 term_scrape({buf}, {row})	List	get row of a terminal screen
 term_sendkeys({buf}, {keys})	none	send keystrokes to a terminal
-term_setapi({buf}, {expr})	none	set |terminal-api| function name prefix
 term_setansicolors({buf}, {colors})
 				none	set ANSI palette in GUI color mode
+term_setapi({buf}, {expr})	none	set |terminal-api| function name prefix
 term_setkill({buf}, {how})	none	set signal to stop job in terminal
 term_setrestore({buf}, {command}) none	set command to restore terminal
 term_setsize({buf}, {rows}, {cols})
@@ -2897,16 +2896,16 @@ test_null_job()			Job	null value for testing
 test_null_list()		List	null value for testing
 test_null_partial()		Funcref	null value for testing
 test_null_string()		String	null value for testing
-test_unknown()			any	unknown value for testing
-test_void()			any	void value for testing
 test_option_not_set({name})	none	reset flag indicating option was set
 test_override({expr}, {val})	none	test with Vim internal overrides
 test_refcount({expr})		Number	get the reference count of {expr}
 test_scrollbar({which}, {value}, {dragging})
 				none	scroll in the GUI for testing
 test_setmouse({row}, {col})	none	set the mouse position for testing
-test_srand_seed([seed])		none	set seed for testing srand()
 test_settime({expr})		none	set current time for testing
+test_srand_seed([seed])		none	set seed for testing srand()
+test_unknown()			any	unknown value for testing
+test_void()			any	void value for testing
 timer_info([{id}])		List	information about timers
 timer_pause({id}, {pause})	none	pause or unpause a timer
 timer_start({time}, {callback} [, {options}])
@@ -4409,7 +4408,7 @@ feedkeys({string} [, {mode}])				*feedkeys()*
 			script continues.
 			Note that if you manage to call feedkeys() while
 			executing commands, thus calling it recursively, then
-			all typehead will be consumed by the last call.
+			all typeahead will be consumed by the last call.
 		'!'	When used with 'x' will not end Insert mode. Can be
 			used in a test when a timer is set to exit Insert mode
 			a little later.  Useful for testing CursorHoldI.
@@ -4445,7 +4444,7 @@ filewritable({file})					*filewritable()*
 		directory, and we can write to it, the result is 2.
 
 		Can also be used as a |method|: >
-			GetName()->filewriteable()
+			GetName()->filewritable()
 
 
 filter({expr1}, {expr2})				*filter()*
@@ -5446,7 +5445,7 @@ getmousepos()						*getmousepos()*
 		"screenrow" and "screencol" are valid, the others are zero.
 
 		When on the status line below a window or the vertical
-		separater right of a window, the "line" and "column" values
+		separator right of a window, the "line" and "column" values
 		are zero.
 
 		When the position is after the text then "column" is the
@@ -6962,7 +6961,7 @@ mapset({mode}, {abbr}, {dict})				*mapset()*
 			call mapset('n', 0, save_map)
 <		Note that if you are going to replace a map in several modes,
 		e.g. with `:map!`, you need to save the mapping for all of
-		them, since they can differe.
+		them, since they can differ.
 
 
 match({expr}, {pat} [, {start} [, {count}]])			*match()*
@@ -8559,8 +8558,10 @@ searchcount([{options}])					*searchcount()*
 		  recompute	|Boolean|	if |TRUE|, recompute the count
 						like |n| or |N| was executed.
 						otherwise returns the last
-						result by |n|, |N|, or this
-						function is returned.
+						computed result (when |n| or
+						|N| was used when "S" is not
+						in 'shortmess', or this
+						function was called).
 						(default: |TRUE|)
 		  pattern	|String|	recompute if this was given
 						and different with |@/|.

runtime/doc/usr_41.txt

@@ -1,4 +1,4 @@
-*usr_41.txt*	For Vim version 8.2.  Last change: 2020 Mar 15
+*usr_41.txt*	For Vim version 8.2.  Last change: 2020 Jun 01
 
 		     VIM USER MANUAL - by Bram Moolenaar
 
@@ -41,6 +41,11 @@ prefer.  And you can use any colon command in it (commands that start with a
 specific file type.  A complicated macro can be defined by a separate Vim
 script file.  You can think of other uses yourself.
 
+Note: if you are familiar with Python, you can find a comparison between
+Python and Vim script here, with pointers to other documents:
+   https://gist.github.com/yegappan/16d964a37ead0979b05e655aa036cad0
+
+
 Let's start with a simple example: >
 
 	:let i = 1
@@ -631,6 +636,7 @@ List manipulation:					*list-functions*
 	deepcopy()		make a full copy of a List
 	filter()		remove selected items from a List
 	map()			change each List item
+	reduce()		reduce a List to a value
 	sort()			sort a List
 	reverse()		reverse the order of a List
 	uniq()			remove copies of repeated adjacent items
@@ -687,6 +693,7 @@ Floating point computation:				*float-functions*
 	sinh()			hyperbolic sine
 	cosh()			hyperbolic cosine
 	tanh()			hyperbolic tangent
+	isinf()			check for infinity
 	isnan()			check for not a number
 
 Other computation:					*bitwise-function*
@@ -695,6 +702,8 @@ Other computation:					*bitwise-function*
 	or()			bitwise OR
 	xor()			bitwise XOR
 	sha256()		SHA-256 hash
+	rand()			get a pseudo-random number
+	srand()			initialize seed used by rand()
 
 Variables:						*var-functions*
 	type()			type of a variable
@@ -744,6 +753,7 @@ Working with text in the current buffer:		*text-functions*
 	prevnonblank()		find previous non-blank line
 	search()		find a match for a pattern
 	searchpos()		find a match for a pattern
+	searchcount()		get number of matches before/after the cursor
 	searchpair()		find the other end of a start/skip/end
 	searchpairpos()		find the other end of a start/skip/end
 	searchdecl()		search for the declaration of a name
@@ -828,9 +838,11 @@ Buffers, windows and the argument list:
 	listener_remove()	remove a listener callback
 	win_findbuf()		find windows containing a buffer
 	win_getid()		get window ID of a window
+	win_gettype()		get type of window
 	win_gotoid()		go to window with ID
 	win_id2tabwin()		get tab and window nr from window ID
 	win_id2win()		get window nr from window ID
+	win_splitmove()		move window to a split of another window
 	getbufinfo()		get a list with buffer information
 	gettabinfo()		get a list with tab page information
 	getwininfo()		get a list with window information
@@ -907,6 +919,7 @@ Interactive:					*interactive-functions*
 	getchar()		get a character from the user
 	getcharmod()		get modifiers for the last typed character
 	getmousepos()		get last known mouse position
+	echoraw()		output characters as-is
 	feedkeys()		put characters in the typeahead queue
 	input()			get a line from the user
 	inputlist()		let the user pick an entry from a list
@@ -948,6 +961,7 @@ Mappings and Menus:			    *mapping-functions*
 	hasmapto()		check if a mapping exists
 	mapcheck()		check if a matching mapping exists
 	maparg()		get rhs of a mapping
+	mapset()		restore a mapping
 	menu_info()		get information about a menu item
 	wildmenumode()		check if the wildmode is active
 
@@ -968,11 +982,13 @@ Testing:				    *test-functions*
 	test_autochdir()	enable 'autochdir' during startup
 	test_override()		test with Vim internal overrides
 	test_garbagecollect_now()   free memory right now
+	test_garbagecollect_soon()  set a flag to free memory soon
 	test_getvalue()		get value of an internal variable
 	test_ignore_error()	ignore a specific error message
 	test_null_blob()	return a null Blob
 	test_null_channel()	return a null Channel
 	test_null_dict()	return a null Dict
+	test_null_function()	return a null Funcref
 	test_null_job()		return a null Job
 	test_null_list()	return a null List
 	test_null_partial()	return a null Partial function
@@ -982,6 +998,10 @@ Testing:				    *test-functions*
 	test_feedinput()	add key sequence to input buffer
 	test_option_not_set()	reset flag indicating option was set
 	test_scrollbar()	simulate scrollbar movement in the GUI
+	test_refcount()		return an expression's reference count
+	test_srand_seed()	set the seed value for srand()
+	test_unknown()		return a value with unknown type
+	test_void()		return a value with void type
 
 Inter-process communication:		    *channel-functions*
 	ch_canread()		check if there is something to read
@@ -993,8 +1013,8 @@ Inter-process communication:		    *channel-functions*
 	ch_readraw()		read a raw message from a channel
 	ch_sendexpr()		send a JSON message over a channel
 	ch_sendraw()		send a raw message over a channel
-	ch_evalexpr()		evaluates an expression over channel
-	ch_evalraw()		evaluates a raw string over channel
+	ch_evalexpr()		evaluate an expression over channel
+	ch_evalraw()		evaluate a raw string over channel
 	ch_status()		get status of a channel
 	ch_getbufnr()		get the buffer number of a channel
 	ch_getjob()		get the job associated with a channel
@@ -1007,7 +1027,7 @@ Inter-process communication:		    *channel-functions*
 	js_encode()		encode an expression to a JSON string
 	js_decode()		decode a JSON string to Vim types
 
-Jobs:		    			        *job-functions*
+Jobs:						*job-functions*
 	job_start()		start a job
 	job_stop()		stop a job
 	job_status()		get the status of a job
@@ -1050,6 +1070,7 @@ Terminal window:				*terminal-functions*
 	term_setkill()		set signal to stop job in a terminal
 	term_setrestore()	set command to restore a terminal
 	term_setsize()		set the size of a terminal
+	term_setapi()		set terminal JSON API function name prefix
 
 Popup window:					*popup-window-functions*
 	popup_create()		create popup centered in the screen
@@ -1068,9 +1089,13 @@ Popup window:					*popup-window-functions*
 	popup_close()		close one popup
 	popup_clear()		close all popups
 	popup_filter_menu()	select from a list of items
-	popup_filter_yesno()	blocks until 'y' or 'n' is pressed
+	popup_filter_yesno()	block until 'y' or 'n' is pressed
 	popup_getoptions()	get current options for a popup
 	popup_getpos()		get actual position and size of a popup
+	popup_findinfo()	get window ID for popup info window
+	popup_findpreview()	get window ID for popup preview window
+	popup_list()		get list of all popup window IDs
+	popup_locate()		get popup window ID from its screen position
 
 Timers:						*timer-functions*
 	timer_start()		create a timer
@@ -1090,8 +1115,27 @@ Prompt Buffer:					*promptbuffer-functions*
 	prompt_setinterrupt()	set interrupt callback for a buffer
 	prompt_setprompt()	set the prompt text for a buffer
 
+Text Properties:				*text-property-functions*
+	prop_add()		attach a property at a position
+	prop_clear()		remove all properties from a line or lines
+	prop_find()		search for a property
+	prop_list()		return a list of all properties in a line
+	prop_remove()		remove a property from a line
+	prop_type_add()		add/define a property type
+	prop_type_change()	change properties of a type
+	prop_type_delete()	remove a text property type
+	prop_type_get()		return the properties of a type
+	prop_type_list()	return a list of all property types
+
+Sound:							*sound-functions*
+	sound_clear()		stop playing all sounds
+	sound_playevent()	play an event's sound
+	sound_playfile()	play a sound file
+	sound_stop()		stop playing a sound
+
 Various:					*various-functions*
 	mode()			get current editing mode
+	state()			get current busy state
 	visualmode()		last visual mode used
 	exists()		check if a variable, function, etc. exists
 	has()			check if a feature is supported in Vim
@@ -1100,6 +1144,9 @@ Various:					*various-functions*
 	did_filetype()		check if a FileType autocommand was used
 	eventhandler()		check if invoked by an event handler
 	getpid()		get process ID of Vim
+	getimstatus()		check if IME status is active
+	interrupt()		interrupt script execution
+	windowsversion()	get MS-Windows version
 
 	libcall()		call a function in an external library
 	libcallnr()		idem, returning a number
@@ -1117,12 +1164,14 @@ Various:					*various-functions*
 
 	wordcount()		get byte/word/char count of buffer
 
-	luaeval()		evaluate Lua expression
+	luaeval()		evaluate |Lua| expression
 	mzeval()		evaluate |MzScheme| expression
 	perleval()		evaluate Perl expression (|+perl|)
 	py3eval()		evaluate Python expression (|+python3|)
 	pyeval()		evaluate Python expression (|+python|)
 	pyxeval()		evaluate |python_x| expression
+	rubyeval()		evaluate |Ruby| expression
+
 	debugbreak()		interrupt a program being debugged
 
 ==============================================================================

src/Make_cyg_ming.mak

@@ -625,7 +625,7 @@ endif
 
 ifeq ($(CHANNEL),yes)
 DEFINES += -DFEAT_JOB_CHANNEL -DFEAT_IPV6
- ifeq ($(shell expr "$(WINVER)" \>= 0x600),1)
+ ifeq ($(shell expr "$$(($(WINVER)))" \>= "$$((0x600))"),1)
 DEFINES += -DHAVE_INET_NTOP
  endif
 endif

src/getchar.c

@@ -2368,7 +2368,8 @@ handle_mapping(
 	    // Skip ":lmap" mappings if keys were mapped.
 	    if (mp->m_keys[0] == tb_c1
 		    && (mp->m_mode & local_State)
-		    && !(mp->m_simplified && seenModifyOtherKeys)
+		    && !(mp->m_simplified && seenModifyOtherKeys
+						     && typebuf.tb_maplen == 0)
 		    && ((mp->m_mode & LANGMAP) == 0 || typebuf.tb_maplen == 0))
 	    {
 #ifdef FEAT_LANGMAP

src/mbyte.c

@@ -3842,6 +3842,158 @@ utf_head_off(char_u *base, char_u *p)
     return (int)(p - q);
 }
 
+/*
+ * Whether space is NOT allowed before/after 'c'.
+ */
+    int
+utf_eat_space(int cc)
+{
+    return ((cc >= 0x2000 && cc <= 0x206F)	// General punctuations
+	 || (cc >= 0x2e00 && cc <= 0x2e7f)	// Supplemental punctuations
+	 || (cc >= 0x3000 && cc <= 0x303f)	// CJK symbols and punctuations
+	 || (cc >= 0xff01 && cc <= 0xff0f)	// Full width ASCII punctuations
+	 || (cc >= 0xff1a && cc <= 0xff20)	// ..
+	 || (cc >= 0xff3b && cc <= 0xff40)	// ..
+	 || (cc >= 0xff5b && cc <= 0xff65));	// ..
+}
+
+/*
+ * Whether line break is allowed before "cc".
+ */
+    int
+utf_allow_break_before(int cc)
+{
+    static const int BOL_prohibition_punct[] =
+    {
+	'!',
+	'%',
+	')',
+	',',
+	':',
+	';',
+	'>',
+	'?',
+	']',
+	'}',
+	0x2019, // ’ right single quotation mark
+	0x201d, // ” right double quotation mark
+	0x2020, // † dagger
+	0x2021, // ‡ double dagger
+	0x2026, // … horizontal ellipsis
+	0x2030, // ‰ per mille sign
+	0x2031, // ‱ per then thousand sign
+	0x203c, // ‼ double exclamation mark
+	0x2047, // ⁇ double question mark
+	0x2048, // ⁈ question exclamation mark
+	0x2049, // ⁉ exclamation question mark
+	0x2103, // ℃ degree celsius
+	0x2109, // ℉ degree fahrenheit
+	0x3001, // 、 ideographic comma
+	0x3002, // 。 ideographic full stop
+	0x3009, // 〉 right angle bracket
+	0x300b, // 》 right double angle bracket
+	0x300d, // 」 right corner bracket
+	0x300f, // 』 right white corner bracket
+	0x3011, // 】 right black lenticular bracket
+	0x3015, // 〕 right tortoise shell bracket
+	0x3017, // 〗 right white lenticular bracket
+	0x3019, // 〙 right white tortoise shell bracket
+	0x301b, // 〛 right white square bracket
+	0xff01, // ！ fullwidth exclamation mark
+	0xff09, // ） fullwidth right parenthesis
+	0xff0c, // ， fullwidth comma
+	0xff0e, // ． fullwidth full stop
+	0xff1a, // ： fullwidth colon
+	0xff1b, // ； fullwidth semicolon
+	0xff1f, // ？ fullwidth question mark
+	0xff3d, // ］ fullwidth right square bracket
+	0xff5d, // ｝ fullwidth right curly bracket
+    };
+
+    int first = 0;
+    int last  = sizeof(BOL_prohibition_punct)/sizeof(int) - 1;
+    int mid   = 0;
+
+    while (first < last)
+    {
+	mid = (first + last)/2;
+
+	if (cc == BOL_prohibition_punct[mid])
+	    return FALSE;
+	else if (cc > BOL_prohibition_punct[mid])
+	    first = mid + 1;
+	else
+	    last = mid - 1;
+    }
+
+    return cc != BOL_prohibition_punct[first];
+}
+
+/*
+ * Whether line break is allowed after "cc".
+ */
+    static int
+utf_allow_break_after(int cc)
+{
+    static const int EOL_prohibition_punct[] =
+    {
+	'(',
+	'<',
+	'[',
+	'`',
+	'{',
+	//0x2014, // — em dash
+	0x2018, // ‘ left single quotation mark
+	0x201c, // “ left double quotation mark
+	//0x2053, // ～ swung dash
+	0x3008, // 〈 left angle bracket
+	0x300a, // 《 left double angle bracket
+	0x300c, // 「 left corner bracket
+	0x300e, // 『 left white corner bracket
+	0x3010, // 【 left black lenticular bracket
+	0x3014, // 〔 left tortoise shell bracket
+	0x3016, // 〖 left white lenticular bracket
+	0x3018, // 〘 left white tortoise shell bracket
+	0x301a, // 〚 left white square bracket
+	0xff08, // （ fullwidth left parenthesis
+	0xff3b, // ［ fullwidth left square bracket
+	0xff5b, // ｛ fullwidth left curly bracket
+    };
+
+    int first = 0;
+    int last  = sizeof(EOL_prohibition_punct)/sizeof(int) - 1;
+    int mid   = 0;
+
+    while (first < last)
+    {
+	mid = (first + last)/2;
+
+	if (cc == EOL_prohibition_punct[mid])
+	    return FALSE;
+	else if (cc > EOL_prohibition_punct[mid])
+	    first = mid + 1;
+	else
+	    last = mid - 1;
+    }
+
+    return cc != EOL_prohibition_punct[first];
+}
+
+/*
+ * Whether line break is allowed between "cc" and "ncc".
+ */
+    int
+utf_allow_break(int cc, int ncc)
+{
+    // don't break between two-letter punctuations
+    if (cc == ncc
+	    && (cc == 0x2014 // em dash
+		|| cc == 0x2026)) // horizontal ellipsis
+	return FALSE;
+
+    return utf_allow_break_after(cc) && utf_allow_break_before(ncc);
+}
+
 /*
  * Copy a character from "*fp" to "*tp" and advance the pointers.
  */

src/ops.c

@@ -1967,7 +1967,10 @@ do_join(
 		    && (!has_format_option(FO_MBYTE_JOIN)
 			|| (mb_ptr2char(curr) < 0x100 && endcurr1 < 0x100))
 		    && (!has_format_option(FO_MBYTE_JOIN2)
-			|| mb_ptr2char(curr) < 0x100 || endcurr1 < 0x100)
+			|| (mb_ptr2char(curr) < 0x100
+			    && !(enc_utf8 && utf_eat_space(endcurr1)))
+			|| (endcurr1 < 0x100
+			    && !(enc_utf8 && utf_eat_space(mb_ptr2char(curr)))))
 	       )
 	    {
 		// don't add a space if the line is ending in a space

src/option.h

@@ -141,12 +141,13 @@
 #define FO_ONE_LETTER	'1'
 #define FO_WHITE_PAR	'w'	// trailing white space continues paragr.
 #define FO_AUTO		'a'	// automatic formatting
+#define FO_RIGOROUS_TW	']'     // respect textwidth rigorously
 #define FO_REMOVE_COMS	'j'	// remove comment leaders when joining lines
 #define FO_PERIOD_ABBR	'p'	// don't break a single space after a period
 
 #define DFLT_FO_VI	"vt"
 #define DFLT_FO_VIM	"tcq"
-#define FO_ALL		"tcroq2vlb1mMBn,awjp"	// for do_set()
+#define FO_ALL		"tcroq2vlb1mMBn,aw]jp"	// for do_set()
 
 // characters for the p_cpo option:
 #define CPO_ALTREAD	'a'	// ":read" sets alternate file name

src/proto/mbyte.pro

@@ -52,6 +52,9 @@ void show_utf8(void);
 int latin_head_off(char_u *base, char_u *p);
 int dbcs_screen_head_off(char_u *base, char_u *p);
 int utf_head_off(char_u *base, char_u *p);
+int utf_eat_space(int cc);
+int utf_allow_break_before(int cc);
+int utf_allow_break(int cc, int ncc);
 void mb_copy_char(char_u **fp, char_u **tp);
 int mb_off_next(char_u *base, char_u *p);
 int mb_tail_off(char_u *base, char_u *p);

src/search.c

@@ -360,10 +360,9 @@ static int	    saved_no_hlsearch = 0;
     void
 save_last_search_pattern(void)
 {
-    if (did_save_last_search_spat != 0)
-	iemsg("did_save_last_search_spat is not zero");
-    else
-	++did_save_last_search_spat;
+    if (++did_save_last_search_spat != 1)
+	// nested call, nothing to do
+	return;
 
     saved_last_search_spat = spats[RE_SEARCH];
     if (spats[RE_SEARCH].pat != NULL)
@@ -375,12 +374,14 @@ save_last_search_pattern(void)
     void
 restore_last_search_pattern(void)
 {
-    if (did_save_last_search_spat != 1)
+    if (--did_save_last_search_spat > 0)
+	// nested call, nothing to do
+	return;
+    if (did_save_last_search_spat != 0)
     {
-	iemsg("did_save_last_search_spat is not one");
+	iemsg("restore_last_search_pattern() called more often than save_last_search_pattern()");
 	return;
     }
-    --did_save_last_search_spat;
 
     vim_free(spats[RE_SEARCH].pat);
     spats[RE_SEARCH] = saved_last_search_spat;

src/testdir/Make_all.mak

@@ -85,6 +85,7 @@ NEW_TESTS = \
 	test_charsearch_utf8 \
 	test_checkpath \
 	test_cindent \
+	test_cjk_linebreak \
 	test_clientserver \
 	test_close_count \
 	test_cmdline \
@@ -139,6 +140,7 @@ NEW_TESTS = \
 	test_fnamemodify \
 	test_fold \
 	test_functions \
+	test_function_lists \
 	test_ga \
 	test_getcwd \
 	test_getvar \
@@ -333,6 +335,7 @@ NEW_TESTS_RES = \
 	test_charsearch.res \
 	test_checkpath.res \
 	test_cindent.res \
+	test_cjk_linebreak.res \
 	test_clientserver.res \
 	test_close_count.res \
 	test_cmdline.res \
@@ -375,6 +378,7 @@ NEW_TESTS_RES = \
 	test_fnameescape.res \
 	test_fold.res \
 	test_functions.res \
+	test_function_lists.res \
 	test_getcwd.res \
 	test_getvar.res \
 	test_gf.res \

src/testdir/dumps/Test_searchstat_4.dump

@@ -0,0 +1,10 @@
+|t+0&#ffffff0|h|i|s| |i|s| |s+1&&|o|m|e|t|h|i|n|g| +0&&@57
+@75
+|~+0#4040ff13&| @73
+|~| @73
+|~| @73
+|~| @73
+|~| @73
+|~| @73
+|[+3#0000000&|1|/|1|]| @69
+|/+0&&|s|o|m|e|t|h|i|n|g> @64

src/testdir/test_assert.vim

@@ -78,11 +78,18 @@ func Test_assert_equalfile()
   call writefile(['1234X89'], 'Xone')
   call writefile(['1234Y89'], 'Xtwo')
   call assert_equal(1, assert_equalfile('Xone', 'Xtwo'))
-  call assert_match("difference at byte 4", v:errors[0])
+  call assert_match('difference at byte 4, line 1 after "1234X" vs "1234Y"', v:errors[0])
+  call remove(v:errors, 0)
+
+  call writefile([repeat('x', 234) .. 'X'], 'Xone')
+  call writefile([repeat('x', 234) .. 'Y'], 'Xtwo')
+  call assert_equal(1, assert_equalfile('Xone', 'Xtwo'))
+  let xes = repeat('x', 134)
+  call assert_match('difference at byte 234, line 1 after "' .. xes .. 'X" vs "' .. xes .. 'Y"', v:errors[0])
   call remove(v:errors, 0)
 
   call assert_equal(1, assert_equalfile('Xone', 'Xtwo', 'a message'))
-  call assert_match("a message: difference at byte 4", v:errors[0])
+  call assert_match("a message: difference at byte 234, line 1 after", v:errors[0])
   call remove(v:errors, 0)
 
   call delete('Xone')

src/testdir/test_cjk_linebreak.vim

@@ -0,0 +1,91 @@
+scriptencoding utf-8
+
+func Run_cjk_linebreak_after()
+  set textwidth=12
+  for punct in [
+        \ '!', '%', ')', ',', ':', ';', '>', '?', ']', '}', '’', '”', '†', '‡',
+        \ '…', '‰', '‱', '‼', '⁇', '⁈', '⁉', '℃', '℉', '、', '。', '〉', '》',
+        \ '」', '』', '】', '〕', '〗', '〙', '〛', '！', '）', '，', '．', '：',
+        \ '；', '？', '］', '｝']
+    call setline('.', '这是一个测试'.punct.'试试 CJK 行禁则补丁。')
+    normal gqq
+    call assert_equal('这是一个测试'.punct, getline(1))
+    %d_
+  endfor
+endfunc
+
+func Test_cjk_linebreak_after()
+  set formatoptions=croqn2mB1j
+  call Run_cjk_linebreak_after()
+endfunc
+
+" TODO: this test fails
+"func Test_cjk_linebreak_after_rigorous()
+"  set formatoptions=croqn2mB1j]
+"  call Run_cjk_linebreak_after()
+"endfunc
+
+func Run_cjk_linebreak_before()
+  set textwidth=12
+  for punct in [
+        \ '(', '<', '[', '`', '{', '‘', '“', '〈', '《', '「', '『', '【', '〔',
+        \ '〖', '〘', '〚', '（', '［', '｛']
+    call setline('.', '这是个测试'.punct.'试试 CJK 行禁则补丁。')
+    normal gqq
+    call assert_equal('这是个测试', getline(1))
+    %d_
+  endfor
+endfunc
+
+func Test_cjk_linebreak_before()
+  set formatoptions=croqn2mB1j
+  call Run_cjk_linebreak_before()
+endfunc
+
+func Test_cjk_linebreak_before_rigorous()
+  set formatoptions=croqn2mB1j]
+  call Run_cjk_linebreak_before()
+endfunc
+
+func Run_cjk_linebreak_nobetween()
+  " …… must not start a line
+  call setline('.', '这是个测试……试试 CJK 行禁则补丁。')
+  set textwidth=12 ambiwidth=double
+  normal gqq
+  " TODO: this fails
+  " call assert_equal('这是个测试……', getline(1))
+  %d_
+
+  call setline('.', '这是一个测试……试试 CJK 行禁则补丁。')
+  set textwidth=12 ambiwidth=double
+  normal gqq
+  call assert_equal('这是一个测', getline(1))
+  %d_
+
+  " but —— can
+  call setline('.', '这是个测试——试试 CJK 行禁则补丁。')
+  set textwidth=12 ambiwidth=double
+  normal gqq
+  call assert_equal('这是个测试', getline(1))
+endfunc
+
+func Test_cjk_linebreak_nobetween()
+  set formatoptions=croqn2mB1j
+  call Run_cjk_linebreak_nobetween()
+endfunc
+
+func Test_cjk_linebreak_nobetween_rigorous()
+  set formatoptions=croqn2mB1j]
+  call Run_cjk_linebreak_nobetween()
+endfunc
+
+func Test_cjk_linebreak_join_punct()
+  for punct in ['——', '〗', '，', '。', '……']
+    call setline(1, '文本文本'.punct)
+    call setline(2, 'English')
+    set formatoptions=croqn2mB1j
+    normal ggJ
+    call assert_equal('文本文本'.punct.'English', getline(1))
+    %d_
+  endfor
+endfunc

src/testdir/test_cmdline.vim

@@ -1220,6 +1220,22 @@ func Test_cmd_bang()
   call delete('Xresult')
 endfunc
 
+" Test error: "E135: *Filter* Autocommands must not change current buffer"
+func Test_cmd_bang_E135()
+  new
+  call setline(1, ['a', 'b', 'c', 'd'])
+  augroup test_cmd_filter_E135
+    au!
+    autocmd FilterReadPost * help
+  augroup END
+  call assert_fails('2,3!echo "x"', 'E135:')
+
+  augroup test_cmd_filter_E135
+    au!
+  augroup END
+  %bwipe!
+endfunc
+
 " Test for using ~ for home directory in cmdline completion matches
 func Test_cmdline_expand_home()
   call mkdir('Xdir')

src/testdir/test_function_lists.vim

@@ -0,0 +1,104 @@
+" Test to verify that the three function lists,
+"
+"   global_functions[] in src/evalfunc.c
+"   *functions* in runtime/doc/eval.txt
+"   *function-list* in runtime/doc/usr_41.txt
+"
+" contain the same functions and that the global_functions and ":help
+" functions" lists are in ASCII order.
+
+func Test_function_lists()
+
+  " Delete any files left over from an earlier run of this test.
+
+  call delete("Xglobal_functions.diff")
+  call delete("Xfunctions.diff")
+  call delete("Xfunction-list.diff")
+
+  " Create a file of the functions in evalfunc.c:global_functions[].
+
+  enew!
+  read ../evalfunc.c
+  1,/^static funcentry_T global_functions\[\] =$/d
+  call search('^};$')
+  .,$d
+  v/^    {/d
+  %s/^    {"//
+  %s/".*//
+  w! Xglobal_functions
+
+  " Verify that those functions are in ASCII order.
+
+  sort u
+  w! Xsorted_global_functions
+  let l:unequal = assert_equalfile("Xsorted_global_functions", "Xglobal_functions",
+      \ "global_functions[] not sorted")
+  if l:unequal && executable("diff")
+    call system("diff -u Xsorted_global_functions Xglobal_functions > Xglobal_functions.diff")
+  endif
+
+  " Create a file of the functions in evalfunc.c:global_functions[] that are
+  " not obsolete, sorted in ASCII order.
+
+  enew!
+  read ../evalfunc.c
+  1,/^static funcentry_T global_functions\[\] =$/d
+  call search('^};$')
+  .,$d
+  v/^    {/d
+  g/\/\/ obsolete$/d
+  %s/^    {"//
+  %s/".*//
+  sort u
+  w! ++ff=unix Xsorted_current_global_functions
+
+  " Verify that the ":help functions" list is complete and in ASCII order.
+
+  enew!
+  read ../../runtime/doc/eval.txt
+  call search('\*functions\*$')
+  call search('^USAGE')
+  1,.d
+  call search('\*\K\k*()\*$')
+  .,$d
+  v/^\S/d
+  %s/(.*//
+  let l:lines = getline(1, '$')
+  call uniq(l:lines)
+  call writefile(l:lines, "Xfunctions")
+  let l:unequal = assert_equalfile("Xsorted_current_global_functions", "Xfunctions",
+      \ "\":help functions\" not sorted or incomplete")
+  if l:unequal && executable("diff")
+    call system("diff -u Xsorted_current_global_functions Xfunctions > Xfunctions.diff")
+  endif
+
+  " Verify that the ":help function-list" list is complete.
+
+  enew!
+  read ../../runtime/doc/usr_41.txt
+  call search('\*function-list\*$')
+  1,.d
+  call search('^==*$')
+  .,$d
+  v/^\t\S/d
+  %s/(.*//
+  %left
+  sort u
+  w! ++ff=unix Xfunction-list
+  let l:unequal = assert_equalfile("Xsorted_current_global_functions", "Xfunction-list",
+      \ "\":help functions-list\" incomplete")
+  if l:unequal && executable("diff")
+    call system("diff -u Xsorted_current_global_functions Xfunction-list > Xfunction-list.diff")
+  endif
+
+  " Clean up.
+  call delete("Xglobal_functions")
+  call delete("Xsorted_global_functions")
+  call delete("Xsorted_current_global_functions")
+  call delete("Xfunctions")
+  call delete("Xfunction-list")
+  enew!
+
+endfunc
+
+" vim: shiftwidth=2 sts=2 expandtab

src/testdir/test_ga.vim

@@ -25,6 +25,7 @@ func Test_ga_command()
   " Test a few multi-bytes characters.
   call assert_equal("\n<é> 233, Hex 00e9, Oct 351, Digr e'",    Do_ga('é'))
   call assert_equal("\n<ẻ> 7867, Hex 1ebb, Oct 17273, Digr e2", Do_ga('ẻ'))
+  call assert_equal("\n<\U00012345> 74565, Hex 00012345, Octal 221505", Do_ga("\U00012345"))
 
   " Test with combining characters.
   call assert_equal("\n<e>  101,  Hex 65,  Octal 145 < ́> 769, Hex 0301, Octal 1401", Do_ga("e\u0301"))

src/testdir/test_gui.vim

@@ -861,4 +861,14 @@ func Test_gui_run_cmd_in_terminal()
   let &guioptions = save_guioptions
 endfunc
 
+func Test_gui_recursive_mapping()
+  nmap ' <C-W>
+  nmap <C-W>a :let didit = 1<CR>
+  call feedkeys("'a", 'xt')
+  call assert_equal(1, didit)
+
+  nunmap '
+  nunmap <C-W>a
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab

src/testdir/test_search_stat.vim

@@ -263,6 +263,34 @@ func Test_searchcount_fails()
   call assert_fails('echo searchcount("boo!")', 'E715:')
 endfunc
 
+func Test_searchcount_in_statusline()
+  CheckScreendump
+
+  let lines =<< trim END
+    set shortmess-=S
+    call append(0, 'this is something')
+    function TestSearchCount() abort
+      let search_count = searchcount()
+      if !empty(search_count)
+	return '[' . search_count.current . '/' . search_count.total . ']'
+      else
+	return ''
+      endif
+    endfunction
+    set hlsearch
+    set laststatus=2 statusline+=%{TestSearchCount()}
+  END
+  call writefile(lines, 'Xsearchstatusline')
+  let buf = RunVimInTerminal('-S Xsearchstatusline', #{rows: 10})
+  call TermWait(buf)
+  call term_sendkeys(buf, "/something")
+  call VerifyScreenDump(buf, 'Test_searchstat_4', {})
+
+  call term_sendkeys(buf, "\<Esc>")
+  call StopVimInTerminal(buf)
+  call delete('Xsearchstatusline')
+endfunc
+
 func Test_search_stat_foldopen()
   CheckScreendump
 

src/testing.c

@@ -309,6 +309,9 @@ assert_equalfile(typval_T *argvars)
     garray_T	ga;
     FILE	*fd1;
     FILE	*fd2;
+    char	line1[200];
+    char	line2[200];
+    int		lineidx = 0;
 
     if (fname1 == NULL || fname2 == NULL)
 	return 0;
@@ -329,8 +332,9 @@ assert_equalfile(typval_T *argvars)
 	}
 	else
 	{
-	    int c1, c2;
-	    long count = 0;
+	    int	    c1, c2;
+	    long    count = 0;
+	    long    linecount = 1;
 
 	    for (;;)
 	    {
@@ -347,13 +351,31 @@ assert_equalfile(typval_T *argvars)
 		    STRCPY(IObuff, "second file is shorter");
 		    break;
 		}
-		else if (c1 != c2)
+		else
 		{
-		    vim_snprintf((char *)IObuff, IOSIZE,
-					      "difference at byte %ld", count);
-		    break;
+		    line1[lineidx] = c1;
+		    line2[lineidx] = c2;
+		    ++lineidx;
+		    if (c1 != c2)
+		    {
+			vim_snprintf((char *)IObuff, IOSIZE,
+					    "difference at byte %ld, line %ld",
+							     count, linecount);
+			break;
+		    }
 		}
 		++count;
+		if (c1 == NL)
+		{
+		    ++linecount;
+		    lineidx = 0;
+		}
+		else if (lineidx + 2 == (int)sizeof(line1))
+		{
+		    mch_memmove(line1, line1 + 100, lineidx - 100);
+		    mch_memmove(line2, line2 + 100, lineidx - 100);
+		    lineidx -= 100;
+		}
 	    }
 	    fclose(fd1);
 	    fclose(fd2);
@@ -372,6 +394,19 @@ assert_equalfile(typval_T *argvars)
 	    ga_concat(&ga, (char_u *)": ");
 	}
 	ga_concat(&ga, IObuff);
+	if (lineidx > 0)
+	{
+	    line1[lineidx] = NUL;
+	    line2[lineidx] = NUL;
+	    ga_concat(&ga, (char_u *)" after \"");
+	    ga_concat(&ga, (char_u *)line1);
+	    if (STRCMP(line1, line2) != 0)
+	    {
+		ga_concat(&ga, (char_u *)"\" vs \"");
+		ga_concat(&ga, (char_u *)line2);
+	    }
+	    ga_concat(&ga, (char_u *)"\"");
+	}
 	assert_error(&ga);
 	ga_clear(&ga);
 	return 1;

src/textformat.c

@@ -45,10 +45,12 @@ internal_format(
     int		c) // character to be inserted (can be NUL)
 {
     int		cc;
+    int		skip_pos;
     int		save_char = NUL;
     int		haveto_redraw = FALSE;
     int		fo_ins_blank = has_format_option(FO_INS_BLANK);
     int		fo_multibyte = has_format_option(FO_MBYTE_BREAK);
+    int		fo_rigor_tw  = has_format_option(FO_RIGOROUS_TW);
     int		fo_white_par = has_format_option(FO_WHITE_PAR);
     int		first_line = TRUE;
     colnr_T	leader_len;
@@ -125,6 +127,7 @@ internal_format(
 
 	curwin->w_cursor.col = startcol;
 	foundcol = 0;
+	skip_pos = 0;
 
 	// Find position to break at.
 	// Stop at first entered white when 'formatoptions' has 'v'
@@ -189,8 +192,11 @@ internal_format(
 		if (curwin->w_cursor.col <= (colnr_T)wantcol)
 		    break;
 	    }
-	    else if (cc >= 0x100 && fo_multibyte)
+	    else if ((cc >= 0x100 || !utf_allow_break_before(cc)) && fo_multibyte)
 	    {
+		int ncc;
+		int allow_break;
+
 		// Break after or before a multi-byte character.
 		if (curwin->w_cursor.col != startcol)
 		{
@@ -199,8 +205,14 @@ internal_format(
 			break;
 		    col = curwin->w_cursor.col;
 		    inc_cursor();
-		    // Don't change end_foundcol if already set.
-		    if (foundcol != curwin->w_cursor.col)
+		    ncc = gchar_cursor();
+
+		    allow_break =
+			(enc_utf8 && utf_allow_break(cc, ncc))
+			|| enc_dbcs;
+
+		    // If we have already checked this position, skip!
+		    if (curwin->w_cursor.col != skip_pos && allow_break)
 		    {
 			foundcol = curwin->w_cursor.col;
 			end_foundcol = foundcol;
@@ -213,6 +225,7 @@ internal_format(
 		if (curwin->w_cursor.col == 0)
 		    break;
 
+		ncc = cc;
 		col = curwin->w_cursor.col;
 
 		dec_cursor();
@@ -220,16 +233,65 @@ internal_format(
 
 		if (WHITECHAR(cc))
 		    continue;		// break with space
-		// Don't break until after the comment leader
+		// Don't break until after the comment leader.
 		if (curwin->w_cursor.col < leader_len)
 		    break;
 
 		curwin->w_cursor.col = col;
+		skip_pos = curwin->w_cursor.col;
 
-		foundcol = curwin->w_cursor.col;
-		end_foundcol = foundcol;
+		allow_break =
+		    (enc_utf8 && utf_allow_break(cc, ncc))
+		    || enc_dbcs;
+
+		// Must handle this to respect line break prohibition.
+		if (allow_break)
+		{
+		    foundcol = curwin->w_cursor.col;
+		    end_foundcol = foundcol;
+		}
 		if (curwin->w_cursor.col <= (colnr_T)wantcol)
-		    break;
+		{
+		    int ncc_allow_break =
+			 (enc_utf8 && utf_allow_break_before(ncc)) || enc_dbcs;
+
+		    if (allow_break)
+			break;
+		    if (!ncc_allow_break && !fo_rigor_tw)
+		    {
+			// Enable at most 1 punct hang outside of textwidth.
+			if (curwin->w_cursor.col == startcol)
+			{
+			    // We are inserting a non-breakable char, postpone
+			    // line break check to next insert.
+			    end_foundcol = foundcol = 0;
+			    break;
+			}
+
+			// Neither cc nor ncc is NUL if we are here, so
+			// it's safe to inc_cursor.
+			col = curwin->w_cursor.col;
+
+			inc_cursor();
+			cc  = ncc;
+			ncc = gchar_cursor();
+			// handle insert
+			ncc = (ncc != NUL) ? ncc : c;
+
+			allow_break =
+				(enc_utf8 && utf_allow_break(cc, ncc))
+				|| enc_dbcs;
+
+			if (allow_break)
+			{
+			    // Break only when we are not at end of line.
+			    end_foundcol = foundcol =
+				      ncc == NUL? 0 : curwin->w_cursor.col;
+			    break;
+			}
+			curwin->w_cursor.col = col;
+		    }
+		}
 	    }
 	    if (curwin->w_cursor.col == 0)
 		break;

src/version.c

@@ -761,6 +761,24 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    905,
+/**/
+    904,
+/**/
+    903,
+/**/
+    902,
+/**/
+    901,
+/**/
+    900,
+/**/
+    899,
+/**/
+    898,
+/**/
+    897,
 /**/
     896,
 /**/