Colour the editor view with relocated semantic tokens

The results document only ever got TextMate colouring, because a language
server will not serve semantic tokens for a `colored-refs` URI: there is no
file and no compilation behind it. So identifiers looked the way a C# file
does before the server has analysed it.

Instead of tokenizing anything ourselves, fetch the tokens for each *source*
file and move them:

- vscode.provideDocumentSemanticTokensLegend / provideDocumentSemanticTokens
  give the legend and the delta-encoded tokens for a real file. Neither is
  listed by getCommands(), so a test calls them to prove they exist rather
  than looking them up.
- Decode to absolute positions, keep the tokens on displayed lines, and shift
  each column by CODE_INDENT minus the stripped leading whitespace — the same
  mapping that places the highlight ranges. Re-encode with the source legend,
  so the type and modifier numbers stay meaningful.
- The legend is only known after a server answers, but
  registerDocumentSemanticTokensProvider wants it up front, so registration
  is deferred to the first search and redone if a later legend differs.
- Runs after the results are on screen and only changes colours, never text,
  so nothing waits on it. coloredReferences.semanticTokens turns it off;
  results over 40 files skip it.

Verified against DotRush: every relocated token covers exactly the text it
covered in the source file, and `Profiler` comes back typed as `class`.

README: drops the claim that semantic tokens would fix the panel's colours.
They supply the classification, never the colours, and webviews are not given
theme token colours — so only the editor view can be theme-exact. Feeding the
panel real token types is now roadmap item 5.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/test/suite/index.ts

@@ -13,6 +13,7 @@ export function run(): Promise<void> {
     mocha.addFile(path.resolve(__dirname, 'references.test.js'));
     mocha.addFile(path.resolve(__dirname, 'panel.test.js'));
     mocha.addFile(path.resolve(__dirname, 'kinds.test.js'));
+    mocha.addFile(path.resolve(__dirname, 'semantic.test.js'));
 
     return new Promise((resolve, reject) => {
         try {

src/test/suite/semantic.test.ts

@@ -0,0 +1,160 @@
+import * as assert from 'assert';
+import * as path from 'path';
+import * as vscode from 'vscode';
+
+import { SemanticOverlay, decodeTokens } from '../../semantic';
+import { CODE_INDENT, gather, renderDocument } from '../../references';
+
+const TARGET_FILE = path.join('Nerfed.Runtime', 'Profiler.cs');
+const TARGET_SYMBOL = 'Profiler';
+const TARGET_DECL = 'public static class Profiler';
+
+const LEGEND_COMMAND = 'vscode.provideDocumentSemanticTokensLegend';
+const TOKENS_COMMAND = 'vscode.provideDocumentSemanticTokens';
+
+suite('Semantic token overlay', () => {
+    let sourceUri: vscode.Uri;
+    let position: vscode.Position;
+
+    suiteSetup(async () => {
+        const ext = vscode.extensions.getExtension('local.colored-references');
+        assert.ok(ext);
+        await ext.activate();
+
+        const folder = vscode.workspace.workspaceFolders?.[0];
+        assert.ok(folder, 'no workspace folder');
+        sourceUri = vscode.Uri.joinPath(folder.uri, ...TARGET_FILE.split(path.sep));
+        const doc = await vscode.workspace.openTextDocument(sourceUri);
+        for (let i = 0; i < doc.lineCount; i++) {
+            const at = doc.lineAt(i).text.indexOf(TARGET_DECL);
+            if (at >= 0) {
+                position = new vscode.Position(i, doc.lineAt(i).text.indexOf(TARGET_SYMBOL, at) + 1);
+                break;
+            }
+        }
+        assert.ok(position, 'declaration not found');
+    });
+
+    test('the built-in semantic token commands are callable', async () => {
+        // These API commands are not enumerated by getCommands(), so the only way to
+        // check they exist is to call them: a missing command rejects with "not found".
+        const commands = await vscode.commands.getCommands(true);
+        assert.ok(!commands.includes(TOKENS_COMMAND),
+            'this assertion documents that the command is unlisted — if it now appears, ' +
+            'simplify this test to a getCommands() check');
+
+        for (const command of [LEGEND_COMMAND, TOKENS_COMMAND]) {
+            try {
+                await vscode.commands.executeCommand(command, sourceUri);
+            } catch (error) {
+                assert.fail(`${command} is not available: ${error}`);
+            }
+        }
+    });
+
+    test('the language server serves a legend and tokens for a real file', async () => {
+        const legend = await vscode.commands.executeCommand<vscode.SemanticTokensLegend>(
+            LEGEND_COMMAND, sourceUri);
+        assert.ok(legend?.tokenTypes?.length, 'no legend from the language server');
+        console.log(`[test] token types: ${legend.tokenTypes.slice(0, 20).join(', ')}` +
+            `${legend.tokenTypes.length > 20 ? ', …' : ''}`);
+
+        const tokens = await vscode.commands.executeCommand<vscode.SemanticTokens>(
+            TOKENS_COMMAND, sourceUri);
+        assert.ok(tokens?.data?.length, 'no semantic tokens for the source file');
+        assert.strictEqual(tokens.data.length % 5, 0, 'token data must be 5-tuples');
+    });
+
+    test('decodeTokens turns deltas back into absolute positions', () => {
+        // two tokens on line 3, one on line 5
+        const data = new Uint32Array([3, 4, 6, 1, 0, 0, 10, 3, 2, 0, 2, 7, 5, 4, 1]);
+        assert.deepStrictEqual(decodeTokens(data), [
+            { line: 3, char: 4, length: 6, type: 1, modifiers: 0 },
+            { line: 3, char: 14, length: 3, type: 2, modifiers: 0 },
+            { line: 5, char: 7, length: 5, type: 4, modifiers: 1 },
+        ]);
+    });
+
+    test('relocated tokens cover the same text in the results document', async () => {
+        const locations = await vscode.commands.executeCommand<vscode.Location[]>(
+            'vscode.executeReferenceProvider', sourceUri, position);
+        assert.ok(locations && locations.length > 5, 'not enough references to test with');
+
+        const results = await gather(
+            TARGET_SYMBOL, 'csharp', { uri: sourceUri, position }, locations, true);
+        const rendered = renderDocument(results);
+
+        const overlay = new SemanticOverlay();
+        const uri = vscode.Uri.from({ scheme: 'colored-refs', path: '/semantic-test.cs', query: 't' });
+        const built = await overlay.build(uri, rendered.lines, results);
+        assert.ok(built, 'the overlay produced no tokens');
+
+        const served = overlay.peek(uri);
+        assert.ok(served, 'no tokens stored for the results document');
+        const tokens = decodeTokens(served.data);
+        assert.ok(tokens.length > 10, `only ${tokens.length} tokens relocated`);
+
+        const displayLines = rendered.content.split('\n');
+        const sources = new Map<string, vscode.TextDocument>();
+        for (const file of results.files) {
+            sources.set(file.uri.toString(), await vscode.workspace.openTextDocument(file.uri));
+        }
+
+        // Every token must sit on a code line and cover exactly the text it covered in
+        // the source file — that is the whole correctness claim of the relocation.
+        const failures: string[] = [];
+        for (const token of tokens) {
+            const line = rendered.lines[token.line];
+            if (!line || line.kind !== 'code' || !line.file || line.sourceLine === undefined) {
+                failures.push(`token at row ${token.line} is not on a code line`);
+                continue;
+            }
+            const source = sources.get(line.file.toString());
+            if (!source) {
+                failures.push(`token at row ${token.line} has no source document`);
+                continue;
+            }
+            const sourceText = source.lineAt(line.sourceLine).text;
+            const shift = CODE_INDENT.length - (sourceText.length - sourceText.trimStart().length);
+
+            const displayed = displayLines[token.line].substr(token.char, token.length);
+            const original = sourceText.substr(token.char - shift, token.length);
+            if (displayed !== original) {
+                failures.push(`row ${token.line} col ${token.char}: results show ` +
+                    `${JSON.stringify(displayed)} but the source has ${JSON.stringify(original)}`);
+            }
+            if (displayed.trim().length === 0) {
+                failures.push(`row ${token.line} col ${token.char}: token covers whitespace`);
+            }
+        }
+        assert.deepStrictEqual(failures.slice(0, 10), [],
+            `${failures.length} mismatched token(s):\n${failures.slice(0, 10).join('\n')}`);
+    });
+
+    test('the searched symbol is classified as a type', async () => {
+        const legend = await vscode.commands.executeCommand<vscode.SemanticTokensLegend>(
+            LEGEND_COMMAND, sourceUri);
+        assert.ok(legend);
+
+        const locations = await vscode.commands.executeCommand<vscode.Location[]>(
+            'vscode.executeReferenceProvider', sourceUri, position);
+        const results = await gather(
+            TARGET_SYMBOL, 'csharp', { uri: sourceUri, position }, locations ?? [], true);
+        const rendered = renderDocument(results);
+
+        const overlay = new SemanticOverlay();
+        const uri = vscode.Uri.from({ scheme: 'colored-refs', path: '/semantic-kinds.cs', query: 't' });
+        assert.ok(await overlay.build(uri, rendered.lines, results));
+        const tokens = decodeTokens(overlay.peek(uri)!.data);
+
+        const displayLines = rendered.content.split('\n');
+        const onSymbol = tokens.filter(t =>
+            displayLines[t.line]?.substr(t.char, t.length) === TARGET_SYMBOL);
+        assert.ok(onSymbol.length > 0, `no token covers '${TARGET_SYMBOL}'`);
+
+        const names = [...new Set(onSymbol.map(t => legend.tokenTypes[t.type]))];
+        console.log(`[test] '${TARGET_SYMBOL}' token types: ${names.join(', ')}`);
+        assert.ok(names.every(n => /class|type|struct|enum|interface/i.test(n)),
+            `'${TARGET_SYMBOL}' is a static class but was typed ${names.join(', ')}`);
+    });
+});