Semantic Tokens Support (#1275)

* initial naive implementation of semanticTokens

* fix race condition when starting new lsp server

* fix error when closing and re-opening a buffer

* support nvim highlighting

* provide tokenTypes and tokenModifiers client capabilities

* move semantic.vim to autoload/lsp/internal

* refactor semantic tokens to use a callbag

* remove scope tree functions

* refactor decoding of semanticTokens responses

* improve semantic tokens nvim support

* add support for multi-byte characters

* update semantic tokens on TextChangedP

* don't apply semantic highlighting to large files

* replace semantic_highlight server key with custom highlight groups

* rename g:lsp_large_buffer_threshold to g:lsp_max_buffer_size

* fix typo: correct LpEvents to LspEvents

* update semantic highlighting documentation

* change has('popupwin') to exists('autoload/lsp/internal/semantic.vimTextChangedP')

* correct lsp_max_buffer_size usage

* add LspSemanticHighlightGroups helper command

* use explicit scope and reorganise decode_tokens

* use robust comparison + single quoted strings in s:init_highlight

* update semantic highlighting on VimEnter

* support semanticTokens/full/delta requests

* request semanticTokens/full on BufNewFile or BufReadPost

* perform full semantic updates on lsp_buffer_enabled

* support SemanticTokensDelta response to textDocument/semanticTokens/full/delta request

* decide between full or delta semanticTokens request based on state

* use prop_add_list to add semantic tokens

* sort semanticTokens edits before applying

* update semantic.vim to conform to google style guide

* update default semantic highlight groups to better match usage

* fix parsing of SemanticTokensOptions

* mention g:lsp_semantic_enabled in vim-lsp-semantic docs

* document g:lsp_semantic_delay

* add support for semantic token modifiers

* sort semantic tokens modifiers alphabetically

* correctly report semantic tokens capabilities

autoload/lsp.vim

@@ -64,6 +64,7 @@ function! lsp#enable() abort
     call lsp#internal#document_highlight#_enable()
     call lsp#internal#diagnostics#_enable()
     call lsp#internal#document_code_action#signs#_enable()
+    call lsp#internal#semantic#_enable()
     call lsp#internal#show_message_request#_enable()
     call lsp#internal#show_message#_enable()
     call lsp#internal#work_done_progress#_enable()
@@ -80,6 +81,7 @@ function! lsp#disable() abort
     call lsp#internal#document_highlight#_disable()
     call lsp#internal#diagnostics#_disable()
     call lsp#internal#document_code_action#signs#_disable()
+    call lsp#internal#semantic#_disable()
     call lsp#internal#show_message_request#_disable()
     call lsp#internal#show_message#_disable()
     call lsp#internal#work_done_progress#_disable()
@@ -548,8 +550,27 @@ function! lsp#default_get_supported_capabilities(server_info) abort
     \       'references': {
     \           'dynamicRegistration': v:false,
     \       },
-    \       'semanticHighlightingCapabilities': {
-    \           'semanticHighlighting': lsp#ui#vim#semantic#is_enabled()
+    \       'semanticTokens': {
+    \           'dynamicRegistration': v:false,
+    \           'requests': {
+    \               'range': v:false,
+    \               'full': lsp#internal#semantic#is_enabled()
+    \                     ? {'delta': v:true}
+    \                     : v:false
+    \
+    \           },
+    \           'tokenTypes': [
+    \               'type', 'class', 'enum', 'interface', 'struct',
+    \               'typeParameter', 'parameter', 'variable', 'property',
+    \               'enumMember', 'event', 'function', 'method', 'macro',
+    \               'keyword', 'modifier', 'comment', 'string', 'number',
+    \               'regexp', 'operator'
+    \           ],
+    \           'tokenModifiers': [],
+    \           'formats': ['relative'],
+    \           'overlappingTokenSupport': v:false,
+    \           'multilineTokenSupport': v:false,
+    \           'serverCancelSupport': v:false
     \       },
     \       'publishDiagnostics': {
     \           'relatedInformation': v:true,
@@ -846,14 +867,7 @@ function! s:on_notification(server_name, id, data, event) abort
     endif
     call lsp#stream(1, l:stream_data) " notify stream before callbacks
 
-    if lsp#client#is_server_instantiated_notification(a:data)
-        if has_key(l:response, 'method')
-            if l:response['method'] ==# 'textDocument/semanticHighlighting'
-                call lsp#ui#vim#semantic#handle_semantic(a:server_name, a:data)
-            endif
-            " NOTE: this is legacy code, use stream instead of handling notifications here
-        endif
-    else
+    if !lsp#client#is_server_instantiated_notification(a:data)
         let l:request = a:data['request']
         let l:method = l:request['method']
         if l:method ==# 'initialize'