LS intergration for semantic tokens highlighting

spx-gui/src/components/editor/code-editor/code-editor.ts

@@ -8,7 +8,7 @@ import type { Runtime } from '@/models/runtime'
 import type { Project } from '@/models/project'
 import { Copilot } from './copilot'
 import { DocumentBase } from './document-base'
-import { SpxLSPClient } from './lsp'
+import { SpxLSPClient, semanticTokenLegend } from './lsp'
 import {
   type ICodeEditorUI,
   type DiagnosticsContext,
@@ -55,7 +55,8 @@ import {
   type CommandArgs,
   getTextDocumentId,
   containsPosition,
-  makeBasicMarkdownString
+  makeBasicMarkdownString,
+  fromMonacoUri
 } from './common'
 import { TextDocument, createTextDocument } from './text-document'
 import { type Monaco } from './monaco'
@@ -542,6 +543,22 @@ export class CodeEditor extends Disposable {
 
   init() {
     this.lspClient.init()
+
+    this.addDisposable(
+      this.monaco.languages.registerDocumentSemanticTokensProvider('spx', {
+        getLegend: () => semanticTokenLegend,
+        provideDocumentSemanticTokens: async (model) => {
+          const tokens = await this.lspClient.textDocumentSemanticTokens({
+            textDocument: fromMonacoUri(model.uri)
+          })
+          if (tokens == null) return { data: new Uint32Array(0) }
+          return { data: new Uint32Array(tokens.data) } // TODO: pass data with array buffer instead of number array
+        },
+        releaseDocumentSemanticTokens() {
+          // do nothing
+        }
+      })
+    )
   }
 
   dispose(): void {

spx-gui/src/components/editor/code-editor/common.ts

@@ -11,6 +11,7 @@ import { Costume } from '@/models/costume'
 import { Animation } from '@/models/animation'
 import { isWidget } from '@/models/widget'
 import { stageCodeFilePaths } from '@/models/stage'
+import type { Monaco, monaco } from './monaco'
 
 export type Position = {
   line: number
@@ -562,3 +563,11 @@ export function textDocumentId2CodeFileName(id: TextDocumentIdentifier) {
     return { en: spriteName, zh: spriteName }
   }
 }
+
+export function fromMonacoUri(uri: monaco.Uri): TextDocumentIdentifier {
+  return { uri: uri.toString() }
+}
+
+export function toMonacoUri(id: TextDocumentIdentifier, monaco: Monaco): monaco.Uri {
+  return monaco.Uri.parse(id.uri)
+}

spx-gui/src/components/editor/code-editor/lsp/index.ts

@@ -163,6 +163,11 @@ export class SpxLSPClient extends Disposable {
     return spxlc.request<lsp.TextEdit[] | null>(lsp.DocumentFormattingRequest.method, params)
   }
 
+  async textDocumentSemanticTokens(params: lsp.SemanticTokensParams): Promise<lsp.SemanticTokens | null> {
+    const spxlc = await this.prepareRequest()
+    return spxlc.request<lsp.SemanticTokens | null>(lsp.SemanticTokensRequest.method, params)
+  }
+
   // Higher-level APIs
 
   async getResourceReferences(textDocument: TextDocumentIdentifier): Promise<ResourceReference[]> {
@@ -200,3 +205,30 @@ export class SpxLSPClient extends Disposable {
     return completionResult as CompletionItem[]
   }
 }
+
+// Keep in sync with `tools/spxls/internal/server/semantic_token.go`
+export const semanticTokenLegend = {
+  tokenTypes: [
+    lsp.SemanticTokenTypes.namespace,
+    lsp.SemanticTokenTypes.type,
+    lsp.SemanticTokenTypes.interface,
+    lsp.SemanticTokenTypes.struct,
+    lsp.SemanticTokenTypes.parameter,
+    lsp.SemanticTokenTypes.variable,
+    lsp.SemanticTokenTypes.property,
+    lsp.SemanticTokenTypes.function,
+    lsp.SemanticTokenTypes.method,
+    lsp.SemanticTokenTypes.keyword,
+    lsp.SemanticTokenTypes.comment,
+    lsp.SemanticTokenTypes.string,
+    lsp.SemanticTokenTypes.number,
+    lsp.SemanticTokenTypes.operator,
+    'label' // since LSP 3.18.0, Not supported by `vscode-languageserver-protocol` yet
+  ],
+  tokenModifiers: [
+    lsp.SemanticTokenModifiers.declaration,
+    lsp.SemanticTokenModifiers.readonly,
+    lsp.SemanticTokenModifiers.static,
+    lsp.SemanticTokenModifiers.defaultLibrary
+  ]
+}

spx-gui/src/components/editor/code-editor/text-document.ts

@@ -11,7 +11,8 @@ import {
   type TextDocumentIdentifier,
   type WordAtPosition,
   type TextEdit,
-  getTextDocumentId
+  getTextDocumentId,
+  toMonacoUri
 } from './common'
 import { toMonacoPosition, toMonacoRange, fromMonacoPosition } from './ui/common'
 import type { Monaco, monaco } from './monaco'
@@ -114,7 +115,11 @@ export class TextDocument
   ) {
     super()
 
-    this.monacoTextModel = monaco.editor.createModel(codeOwner.getCode() ?? '', 'spx')
+    this.monacoTextModel = monaco.editor.createModel(
+      codeOwner.getCode() ?? '',
+      'spx',
+      toMonacoUri(this.codeOwner.getTextDocumentId(), monaco)
+    )
 
     this.addDisposer(
       watch(

spx-gui/src/components/editor/code-editor/ui/CodeEditorUI.vue

@@ -128,7 +128,9 @@ const monacoEditorOptions = computed<monaco.editor.IStandaloneEditorConstruction
   tabSize,
   insertSpaces,
   fontSize: fontSize.value,
-  contextmenu: false
+  contextmenu: false,
+  // TODO: preserve `semanticHighlighting` of theme (in `@shikijs/monaco`), then remove this line
+  'semanticHighlighting.enabled': true
 }))
 
 const monacEditorInitDataRef = shallowRef<MonacoEditorInitData | null>(null)

spx-gui/src/components/editor/code-editor/ui/common.ts

@@ -4,8 +4,7 @@ import type { ResourceModel } from '@/models/common/resource-model'
 import { Sprite } from '@/models/sprite'
 import { Sound } from '@/models/sound'
 import { isWidget } from '@/models/widget'
-import { type Range, type Position, type TextDocumentIdentifier, type Selection } from '../common'
-import type { Monaco } from '../monaco'
+import { type Range, type Position, type Selection } from '../common'
 
 export function token2Signal(token: monaco.CancellationToken): AbortSignal {
   const ctrl = new AbortController()
@@ -60,15 +59,6 @@ export function toMonacoSelection(selection: Selection): monaco.ISelection {
   }
 }
 
-export function fromMonacoUri(uri: monaco.Uri): TextDocumentIdentifier {
-  // TODO: check if this is correct
-  return { uri: uri.toString() }
-}
-
-export function toMonacoUri(id: TextDocumentIdentifier, monaco: Monaco): monaco.Uri {
-  return monaco.Uri.parse(id.uri)
-}
-
 export function supportGoTo(resourceModel: ResourceModel): boolean {
   // Currently, we do not support "go to detail" for other types of resources due to two reasons:
   // 1. The "selected" state of certain resource types, such as animations, is still managed within the Component, making it difficult to control from here.

spx-gui/src/utils/spx/gop-tm-language.json

@@ -5,9 +5,6 @@
 	"name": "GoPlus",
 	"scopeName": "source.gop",
 	"patterns": [
-		{
-			"include": "#statements"
-		}
 	],
 	"repository": {
 		"statements": {