add links provider for prompt files (#238427)

* [prompt links]: add link background decorations

* [prompt links]: cleanup and move files around

* [prompt links]: add II of PromptLinkProvider, ObjectCache and TrckedDisposable utilities

* [prompt links]: add all missing unit tests

* [prompt links]: remove `TextModelDecoratorsProvider`

* [prompt links]: address PR feedback

src/vs/base/common/assert.ts

@@ -29,9 +29,25 @@ export function assertNever(value: never, message = 'Unreachable'): never {
 	throw new Error(message);
 }
 
-export function assert(condition: boolean, message = 'unexpected state'): asserts condition {
+/**
+ * Asserts that a condition is `truthy`.
+ *
+ * @throws provided {@linkcode messageOrError} if the {@linkcode condition} is `falsy`.
+ *
+ * @param condition The condition to assert.
+ * @param messageOrError An error message or error object to throw if condition is `falsy`.
+ */
+export function assert(
+	condition: boolean,
+	messageOrError: string | Error = 'unexpected state',
+): asserts condition {
 	if (!condition) {
-		throw new BugIndicatingError(`Assertion Failed: ${message}`);
+		// if error instance is provided, use it, otherwise create a new one
+		const errorToThrow = typeof messageOrError === 'string'
+			? new BugIndicatingError(`Assertion Failed: ${messageOrError}`)
+			: messageOrError;
+
+		throw errorToThrow;
 	}
 }
 

src/vs/base/common/codecs/asyncDecoder.ts

@@ -69,7 +69,7 @@ export class AsyncDecoder<T extends NonNullable<unknown>, K extends NonNullable<
 			}
 
 			// if no data available and stream ended, we're done
-			if (this.decoder.isEnded) {
+			if (this.decoder.ended) {
 				this.dispose();
 
 				return null;

src/vs/base/common/codecs/baseDecoder.ts

@@ -7,8 +7,9 @@ import { assert } from '../assert.js';
 import { Emitter } from '../event.js';
 import { IDisposable } from '../lifecycle.js';
 import { ReadableStream } from '../stream.js';
+import { DeferredPromise } from '../async.js';
 import { AsyncDecoder } from './asyncDecoder.js';
-import { TrackedDisposable } from '../trackedDisposable.js';
+import { ObservableDisposable } from '../observableDisposable.js';
 
 /**
  * Event names of {@link ReadableStream} stream.
@@ -24,21 +25,27 @@ export type TStreamListenerNames = 'data' | 'error' | 'end';
 export abstract class BaseDecoder<
 	T extends NonNullable<unknown>,
 	K extends NonNullable<unknown> = NonNullable<unknown>,
-> extends TrackedDisposable implements ReadableStream<T> {
+> extends ObservableDisposable implements ReadableStream<T> {
 	/**
-	 * Flag that indicates if the decoder stream has ended.
+	 * Private attribute to track if the stream has ended.
 	 */
-	protected ended = false;
+	private _ended = false;
 
 	protected readonly _onData = this._register(new Emitter<T>());
-	protected readonly _onEnd = this._register(new Emitter<void>());
-	protected readonly _onError = this._register(new Emitter<Error>());
+	private readonly _onEnd = this._register(new Emitter<void>());
+	private readonly _onError = this._register(new Emitter<Error>());
 
 	/**
 	 * A store of currently registered event listeners.
 	 */
 	private readonly _listeners: Map<TStreamListenerNames, Map<Function, IDisposable>> = new Map();
 
+	/**
+	 * This method is called when a new incomming data
+	 * is received from the input stream.
+	 */
+	protected abstract onStreamData(data: K): void;
+
 	/**
 	 * @param stream The input stream to decode.
 	 */
@@ -53,10 +60,41 @@ export abstract class BaseDecoder<
 	}
 
 	/**
-	 * This method is called when a new incomming data
-	 * is received from the input stream.
+	 * Private attribute to track if the stream has started.
 	 */
-	protected abstract onStreamData(data: K): void;
+	private started = false;
+
+	/**
+	 * Promise that resolves when the stream has ended, either by
+	 * receiving the `end` event or by a disposal, but not when
+	 * the `error` event is received alone.
+	 */
+	private settledPromise = new DeferredPromise<void>();
+
+	/**
+	 * Promise that resolves when the stream has ended, either by
+	 * receiving the `end` event or by a disposal, but not when
+	 * the `error` event is received alone.
+	 *
+	 * @throws If the stream was not yet started to prevent this
+	 * 		   promise to block the consumer calls indefinitely.
+	 */
+	public get settled(): Promise<void> {
+		// if the stream has not started yet, the promise might
+		// block the consumer calls indefinitely if they forget
+		// to call the `start()` method, or if the call happens
+		// after await on the `settled` promise; to forbid this
+		// confusion, we require the stream to be started first
+		assert(
+			this.started,
+			[
+				'Cannot get `settled` promise of a stream that has not been started.',
+				'Please call `start()` first.',
+			].join(' '),
+		);
+
+		return this.settledPromise.p;
+	}
 
 	/**
 	 * Start receiveing data from the stream.
@@ -64,15 +102,20 @@ export abstract class BaseDecoder<
 	 */
 	public start(): this {
 		assert(
-			!this.ended,
+			!this._ended,
 			'Cannot start stream that has already ended.',
 		);
-
 		assert(
 			!this.disposed,
 			'Cannot start stream that has already disposed.',
 		);
 
+		// if already started, nothing to do
+		if (this.started) {
+			return this;
+		}
+		this.started = true;
+
 		this.stream.on('data', this.tryOnStreamData);
 		this.stream.on('error', this.onStreamError);
 		this.stream.on('end', this.onStreamEnd);
@@ -91,8 +134,8 @@ export abstract class BaseDecoder<
 	 * Check if the decoder has been ended hence has
 	 * no more data to produce.
 	 */
-	public get isEnded(): boolean {
-		return this.ended;
+	public get ended(): boolean {
+		return this._ended;
 	}
 
 	/**
@@ -263,12 +306,13 @@ export abstract class BaseDecoder<
 	 * This method is called when the input stream ends.
 	 */
 	protected onStreamEnd(): void {
-		if (this.ended) {
+		if (this._ended) {
 			return;
 		}
 
-		this.ended = true;
+		this._ended = true;
 		this._onEnd.fire();
+		this.settledPromise.complete();
 	}
 
 	/**
@@ -286,7 +330,7 @@ export abstract class BaseDecoder<
 	 */
 	public async consumeAll(): Promise<T[]> {
 		assert(
-			!this.ended,
+			!this._ended,
 			'Cannot consume all messages of the stream that has already ended.',
 		);
 
@@ -309,7 +353,7 @@ export abstract class BaseDecoder<
 	 */
 	[Symbol.asyncIterator](): AsyncIterator<T | null> {
 		assert(
-			!this.ended,
+			!this._ended,
 			'Cannot iterate on messages of the stream that has already ended.',
 		);
 

src/vs/base/common/objectCache.ts

@@ -0,0 +1,152 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Disposable, DisposableMap } from '../../base/common/lifecycle.js';
+import { ObservableDisposable, assertNotDisposed } from './observableDisposable.js';
+
+/**
+ * Generic cache for object instances. Guarantees to return only non-disposed
+ * objects from the {@linkcode get} method. If a requested object is not yet
+ * in the cache or is disposed already, the {@linkcode factory} callback is
+ * called to create a new object.
+ *
+ * @throws if {@linkcode factory} callback returns a disposed object.
+ *
+ * ## Examples
+ *
+ * ```typescript
+ * // a class that will be used as a cache key; the key can be of any
+ * // non-nullable type, including primitives like `string` or `number`,
+ * // but in this case we use an object pointer as a key
+ * class KeyObject {}
+ *
+ * // a class for testing purposes
+ * class TestObject extends ObservableDisposable {
+ *   constructor(
+ *     public readonly id: KeyObject,
+ *   ) {}
+ * };
+ *
+ * // create an object cache instance providing it a factory function that
+ * // is responsible for creating new objects based on the provided key if
+ * // the cache does not contain the requested object yet or an existing
+ * // object is already disposed
+ * const cache = new ObjectCache<TestObject, KeyObject>((key) => {
+ *   // create a new test object based on the provided key
+ *   return new TestObject(key);
+ * });
+ *
+ * // create two keys
+ * const key1 = new KeyObject();
+ * const key2 = new KeyObject();
+ *
+ * // get an object from the cache by its key
+ * const object1 = cache.get(key1); // returns a new test object
+ *
+ * // validate that the new object has the correct key
+ * assert(
+ *   object1.id === key1,
+ *   'Object 1 must have correct ID.',
+ * );
+ *
+ * // returns the same cached test object
+ * const object2 = cache.get(key1);
+ *
+ * // validate that the same exact object is returned from the cache
+ * assert(
+ *   object1 === object2,
+ *   'Object 2 the same cached object as object 1.',
+ * );
+ *
+ * // returns a new test object
+ * const object3 = cache.get(key2);
+ *
+ * // validate that the new object has the correct key
+ * assert(
+ *   object3.id === key2,
+ *   'Object 3 must have correct ID.',
+ * );
+ *
+ * assert(
+ *   object3 !== object1,
+ *   'Object 3 must be a new object.',
+ * );
+ * ```
+ */
+export class ObjectCache<
+	TValue extends ObservableDisposable,
+	TKey extends NonNullable<unknown> = string,
+> extends Disposable {
+	private readonly cache: DisposableMap<TKey, TValue> =
+		this._register(new DisposableMap());
+
+	constructor(
+		private readonly factory: (key: TKey) => TValue & { disposed: false },
+	) {
+		super();
+	}
+
+	/**
+	 * Get an existing object from the cache. If a requested object is not yet
+	 * in the cache or is disposed already, the {@linkcode factory} callback is
+	 * called to create a new object.
+	 *
+	 * @throws if {@linkcode factory} callback returns a disposed object.
+	 * @param key - ID of the object in the cache
+	 */
+	public get(key: TKey): TValue & { disposed: false } {
+		let object = this.cache.get(key);
+
+		// if object is already disposed, remove it from the cache
+		if (object?.disposed) {
+			this.cache.deleteAndLeak(key);
+			object = undefined;
+		}
+
+		// if object exists and is not disposed, return it
+		if (object) {
+			// must always hold true due to the check above
+			assertNotDisposed(
+				object,
+				'Object must not be disposed.',
+			);
+
+			return object;
+		}
+
+		// create a new object by calling the factory
+		object = this.factory(key);
+
+		// newly created object must not be disposed
+		assertNotDisposed(
+			object,
+			'Newly created object must not be disposed.',
+		);
+
+		// remove it from the cache automatically on dispose
+		object.onDispose(() => {
+			this.cache.deleteAndLeak(key);
+		});
+		this.cache.set(key, object);
+
+		return object;
+	}
+
+	/**
+	 * Remove an object from the cache by its key.
+	 *
+	 * @param key ID of the object to remove.
+	 * @param dispose Whether the removed object must be disposed.
+	 */
+	public remove(key: TKey, dispose: boolean): this {
+		if (dispose) {
+			this.cache.deleteAndDispose(key);
+			return this;
+		}
+
+		this.cache.deleteAndLeak(key);
+		return this;
+	}
+}

src/vs/base/common/observableDisposable.ts

@@ -0,0 +1,103 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Emitter } from './event.js';
+import { Disposable } from './lifecycle.js';
+
+/**
+ * Disposable object that tracks its {@linkcode disposed} state
+ * as a public attribute and provides the {@linkcode onDispose}
+ * event to subscribe to.
+ */
+export abstract class ObservableDisposable extends Disposable {
+	/**
+	 * Private emitter for the `onDispose` event.
+	 */
+	private readonly _onDispose = this._register(new Emitter<void>());
+
+	/**
+	 * The event is fired when this object is disposed.
+	 * Note! Executes the callback immediately if already disposed.
+	 *
+	 * @param callback The callback function to be called on updates.
+	 */
+	public onDispose(callback: () => void): this {
+		// if already disposed, execute the callback immediately
+		if (this.disposed) {
+			callback();
+
+			return this;
+		}
+
+		// otherwise subscribe to the event
+		this._register(this._onDispose.event(callback));
+		return this;
+	}
+
+	/**
+	 * Tracks disposed state of this object.
+	 */
+	private _disposed = false;
+
+	/**
+	 * Check if the current object was already disposed.
+	 */
+	public get disposed(): boolean {
+		return this._disposed;
+	}
+
+	/**
+	 * Dispose current object if not already disposed.
+	 * @returns
+	 */
+	public override dispose(): void {
+		if (this.disposed) {
+			return;
+		}
+
+		this._disposed = true;
+		this._onDispose.fire();
+		super.dispose();
+	}
+
+	/**
+	 * Assert that the current object was not yet disposed.
+	 *
+	 * @throws If the current object was already disposed.
+	 * @param error Error message or error object to throw if assertion fails.
+	 */
+	public assertNotDisposed(
+		error: string | Error,
+	): asserts this is TNotDisposed<this> {
+		assertNotDisposed(this, error);
+	}
+}
+
+/**
+ * Type for a non-disposed object `TObject`.
+ */
+type TNotDisposed<TObject extends { disposed: boolean }> = TObject & { disposed: false };
+
+/**
+ * Asserts that a provided `object` is not `disposed` yet,
+ * e.g., its `disposed` property is `false`.
+ *
+ * @throws if the provided `object.disposed` equal to `false`.
+ * @param error Error message or error object to throw if assertion fails.
+ */
+export function assertNotDisposed<TObject extends { disposed: boolean }>(
+	object: TObject,
+	error: string | Error,
+): asserts object is TNotDisposed<TObject> {
+	if (!object.disposed) {
+		return;
+	}
+
+	const errorToThrow = typeof error === 'string'
+		? new Error(error)
+		: error;
+
+	throw errorToThrow;
+}

src/vs/base/common/trackedDisposable.ts

@@ -1,37 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-
-import { Disposable } from '../../base/common/lifecycle.js';
-
-/**
- * Disposable class that tracks its own {@linkcode disposed} state.
- */
-export class TrackedDisposable extends Disposable {
-
-	/**
-	 * Tracks disposed state of this object.
-	 */
-	private _disposed = false;
-
-	/**
-	 * Check if the current object was already disposed.
-	 */
-	public get disposed(): boolean {
-		return this._disposed;
-	}
-
-	/**
-	 * Dispose current object if not already disposed.
-	 * @returns
-	 */
-	public override dispose(): void {
-		if (this.disposed) {
-			return;
-		}
-
-		this._disposed = true;
-		super.dispose();
-	}
-}

src/vs/base/test/common/assert.test.ts

@@ -4,8 +4,9 @@
  *--------------------------------------------------------------------------------------------*/
 
 import assert from 'assert';
-import { ok } from '../../common/assert.js';
+import { ok, assert as commonAssert } from '../../common/assert.js';
 import { ensureNoDisposablesAreLeakedInTestSuite } from './utils.js';
+import { CancellationError, ReadonlyError } from '../../common/errors.js';
 
 suite('Assert', () => {
 	test('ok', () => {
@@ -33,5 +34,70 @@ suite('Assert', () => {
 		ok(5);
 	});
 
+	suite('throws a provided error object', () => {
+		test('generic error', () => {
+			const originalError = new Error('Oh no!');
+
+			try {
+				commonAssert(
+					false,
+					originalError,
+				);
+			} catch (thrownError) {
+				assert.strictEqual(
+					thrownError,
+					originalError,
+					'Must throw the provided error instance.',
+				);
+
+				assert.strictEqual(
+					thrownError.message,
+					'Oh no!',
+					'Must throw the provided error instance.',
+				);
+			}
+		});
+
+		test('cancellation error', () => {
+			const originalError = new CancellationError();
+
+			try {
+				commonAssert(
+					false,
+					originalError,
+				);
+			} catch (thrownError) {
+				assert.strictEqual(
+					thrownError,
+					originalError,
+					'Must throw the provided error instance.',
+				);
+			}
+		});
+
+		test('readonly error', () => {
+			const originalError = new ReadonlyError('World');
+
+			try {
+				commonAssert(
+					false,
+					originalError,
+				);
+			} catch (thrownError) {
+				assert.strictEqual(
+					thrownError,
+					originalError,
+					'Must throw the provided error instance.',
+				);
+
+				assert.strictEqual(
+					thrownError.message,
+					'World is read-only and cannot be changed',
+					'Must throw the provided error instance.',
+				);
+			}
+		});
+	});
+
 	ensureNoDisposablesAreLeakedInTestSuite();
 });

src/vs/base/test/common/objectCache.test.ts

@@ -0,0 +1,332 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert from 'assert';
+import { spy } from 'sinon';
+import { ObjectCache } from '../../common/objectCache.js';
+import { wait } from '../../../base/test/common/testUtils.js';
+import { ObservableDisposable } from '../../common/observableDisposable.js';
+import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../base/test/common/utils.js';
+
+/**
+ * Test object class.
+ */
+class TestObject<TKey extends NonNullable<unknown> = string> extends ObservableDisposable {
+	constructor(
+		public readonly ID: TKey,
+	) {
+		super();
+	}
+
+	/**
+	 * Check if this object is equal to another one.
+	 */
+	public equal(other: TestObject<NonNullable<unknown>>): boolean {
+		return this.ID === other.ID;
+	}
+}
+
+suite('ObjectCache', function () {
+	const disposables = ensureNoDisposablesAreLeakedInTestSuite();
+
+	suite('get', () => {
+		/**
+		 * Common test funtion to test core logic of the cache
+		 * with provider test ID keys of some specific type.
+		 *
+		 * @param key1 Test key1.
+		 * @param key2 Test key2.
+		 */
+		const testCoreLogic = async <TKey extends NonNullable<unknown>>(key1: TKey, key2: TKey) => {
+			const factory = spy((
+				key: TKey,
+			) => {
+				const result: TestObject<TKey> = new TestObject(key);
+
+				result.assertNotDisposed(
+					'Object must not be disposed.',
+				);
+
+				return result;
+			});
+
+			const cache = disposables.add(new ObjectCache(factory));
+
+			/**
+			 * Test the core logic of the cache using 2 objects.
+			 */
+
+			const obj1 = cache.get(key1);
+			assert(
+				factory.calledOnceWithExactly(key1),
+				'[obj1] Must be called once with the correct arguments.',
+			);
+
+			assert(
+				obj1.ID === key1,
+				'[obj1] Returned object must have the correct ID.',
+			);
+
+			const obj2 = cache.get(key1);
+			assert(
+				factory.calledOnceWithExactly(key1),
+				'[obj2] Must be called once with the correct arguments.',
+			);
+
+			assert(
+				obj2.ID === key1,
+				'[obj2] Returned object must have the correct ID.',
+			);
+
+			assert(
+				obj1 === obj2 && obj1.equal(obj2),
+				'[obj2] Returned object must be the same instance.',
+			);
+
+			factory.resetHistory();
+
+			const obj3 = cache.get(key2);
+			assert(
+				factory.calledOnceWithExactly(key2),
+				'[obj3] Must be called once with the correct arguments.',
+			);
+
+			assert(
+				obj3.ID === key2,
+				'[obj3] Returned object must have the correct ID.',
+			);
+
+			factory.resetHistory();
+
+			const obj4 = cache.get(key1);
+			assert(
+				factory.notCalled,
+				'[obj4] Factory must not be called.',
+			);
+
+			assert(
+				obj4.ID === key1,
+				'[obj4] Returned object must have the correct ID.',
+			);
+
+			assert(
+				obj1 === obj4 && obj1.equal(obj4),
+				'[obj4] Returned object must be the same instance.',
+			);
+
+			factory.resetHistory();
+
+			/**
+			 * Now test that the object is removed automatically from
+			 * the cache when it is disposed.
+			 */
+
+			obj3.dispose();
+			// the object is removed from the cache asynchronously
+			// so add a small delay to ensure the object is removed
+			await wait(5);
+
+			const obj5 = cache.get(key1);
+			assert(
+				factory.notCalled,
+				'[obj5] Factory must not be called.',
+			);
+
+			assert(
+				obj5.ID === key1,
+				'[obj5] Returned object must have the correct ID.',
+			);
+
+			assert(
+				obj1 === obj5 && obj1.equal(obj5),
+				'[obj5] Returned object must be the same instance.',
+			);
+
+			factory.resetHistory();
+
+			/**
+			 * Test that the previously disposed object is recreated
+			 * on the new retrieval call.
+			 */
+
+			const obj6 = cache.get(key2);
+			assert(
+				factory.calledOnceWithExactly(key2),
+				'[obj6] Must be called once with the correct arguments.',
+			);
+
+			assert(
+				obj6.ID === key2,
+				'[obj6] Returned object must have the correct ID.',
+			);
+		};
+
+		test('strings as keys', async function () {
+			await testCoreLogic('key1', 'key2');
+		});
+
+		test('numbers as keys', async function () {
+			await testCoreLogic(10, 17065);
+		});
+
+		test('objects as keys', async function () {
+			await testCoreLogic(
+				disposables.add(new TestObject({})),
+				disposables.add(new TestObject({})),
+			);
+		});
+	});
+
+	suite('remove', () => {
+		/**
+		 * Common test funtion to test remove logic of the cache
+		 * with provider test ID keys of some specific type.
+		 *
+		 * @param key1 Test key1.
+		 * @param key2 Test key2.
+		 */
+		const testRemoveLogic = async <TKey extends NonNullable<unknown>>(
+			key1: TKey,
+			key2: TKey,
+			disposeOnRemove: boolean,
+		) => {
+			const factory = spy((
+				key: TKey,
+			) => {
+				const result: TestObject<TKey> = new TestObject(key);
+
+				result.assertNotDisposed(
+					'Object must not be disposed.',
+				);
+
+				return result;
+			});
+
+			// ObjectCache<TestObject<TKey>, TKey>
+			const cache = disposables.add(new ObjectCache(factory));
+
+			/**
+			 * Test the core logic of the cache.
+			 */
+
+			const obj1 = cache.get(key1);
+			assert(
+				factory.calledOnceWithExactly(key1),
+				'[obj1] Must be called once with the correct arguments.',
+			);
+
+			assert(
+				obj1.ID === key1,
+				'[obj1] Returned object must have the correct ID.',
+			);
+
+			factory.resetHistory();
+
+			const obj2 = cache.get(key2);
+			assert(
+				factory.calledOnceWithExactly(key2),
+				'[obj2] Must be called once with the correct arguments.',
+			);
+
+			assert(
+				obj2.ID === key2,
+				'[obj2] Returned object must have the correct ID.',
+			);
+
+			cache.remove(key2, disposeOnRemove);
+
+			const object2Disposed = obj2.disposed;
+
+			// ensure we don't leak undisposed object in the tests
+			if (!obj2.disposed) {
+				obj2.dispose();
+			}
+
+			assert(
+				object2Disposed === disposeOnRemove,
+				`[obj2] Removed object must be disposed: ${disposeOnRemove}.`,
+			);
+
+			factory.resetHistory();
+
+			/**
+			 * Validate that another object is not disposed.
+			 */
+
+			assert(
+				!obj1.disposed,
+				'[obj1] Object must not be disposed.',
+			);
+
+			const obj3 = cache.get(key1);
+			assert(
+				factory.notCalled,
+				'[obj3] Factory must not be called.',
+			);
+
+			assert(
+				obj3.ID === key1,
+				'[obj3] Returned object must have the correct ID.',
+			);
+
+			assert(
+				obj1 === obj3 && obj1.equal(obj3),
+				'[obj3] Returned object must be the same instance.',
+			);
+
+			factory.resetHistory();
+		};
+
+		test('strings as keys', async function () {
+			await testRemoveLogic('key1', 'key2', false);
+			await testRemoveLogic('some-key', 'another-key', true);
+		});
+
+		test('numbers as keys', async function () {
+			await testRemoveLogic(7, 2400700, false);
+			await testRemoveLogic(1090, 2654, true);
+		});
+
+		test('objects as keys', async function () {
+			await testRemoveLogic(
+				disposables.add(new TestObject(1)),
+				disposables.add(new TestObject(1)),
+				false,
+			);
+
+			await testRemoveLogic(
+				disposables.add(new TestObject(2)),
+				disposables.add(new TestObject(2)),
+				true,
+			);
+		});
+	});
+
+	test('throws if factory returns a disposed object', async function () {
+		const factory = (
+			key: string,
+		) => {
+			const result = new TestObject(key);
+
+			if (key === 'key2') {
+				result.dispose();
+			}
+
+			// caution! explicit type casting below!
+			return result as TestObject<string> & { disposed: false };
+		};
+
+		// ObjectCache<TestObject>
+		const cache = disposables.add(new ObjectCache(factory));
+
+		assert.doesNotThrow(() => {
+			cache.get('key1');
+		});
+
+		assert.throws(() => {
+			cache.get('key2');
+		});
+	});
+});

src/vs/base/test/common/observableDisposable.test.ts

@@ -0,0 +1,222 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import assert from 'assert';
+import { spy } from 'sinon';
+import { wait, waitRandom } from './testUtils.js';
+import { Disposable } from '../../common/lifecycle.js';
+import { ensureNoDisposablesAreLeakedInTestSuite } from './utils.js';
+import { assertNotDisposed, ObservableDisposable } from '../../common/observableDisposable.js';
+
+suite('ObservableDisposable', () => {
+	const disposables = ensureNoDisposablesAreLeakedInTestSuite();
+
+	test('tracks `disposed` state', () => {
+		// this is an abstract class, so we have to create
+		// an anonymous class that extends it
+		const object = new class extends ObservableDisposable { }();
+		disposables.add(object);
+
+		assert(
+			object instanceof ObservableDisposable,
+			'Object must be instance of ObservableDisposable.',
+		);
+
+		assert(
+			object instanceof Disposable,
+			'Object must be instance of Disposable.',
+		);
+
+		assert(
+			!object.disposed,
+			'Object must not be disposed yet.',
+		);
+
+		object.dispose();
+
+		assert(
+			object.disposed,
+			'Object must be disposed.',
+		);
+	});
+
+	suite('onDispose', () => {
+		test('fires the event on dispose', async () => {
+			// this is an abstract class, so we have to create
+			// an anonymous class that extends it
+			const object = new class extends ObservableDisposable { }();
+			disposables.add(object);
+
+			assert(
+				!object.disposed,
+				'Object must not be disposed yet.',
+			);
+
+			const onDisposeSpy = spy(() => { });
+			object.onDispose(onDisposeSpy);
+
+			assert(
+				onDisposeSpy.notCalled,
+				'`onDispose` callback must not be called yet.',
+			);
+
+			await waitRandom(10);
+
+			assert(
+				onDisposeSpy.notCalled,
+				'`onDispose` callback must not be called yet.',
+			);
+
+			// dispose object and wait for the event to be fired/received
+			object.dispose();
+			await wait(1);
+
+			/**
+			 * Validate that the callback was called.
+			 */
+
+			assert(
+				object.disposed,
+				'Object must be disposed.',
+			);
+
+			assert(
+				onDisposeSpy.calledOnce,
+				'`onDispose` callback must be called.',
+			);
+
+			/**
+			 * Validate that the callback is not called again.
+			 */
+
+			object.dispose();
+			object.dispose();
+			await waitRandom(10);
+			object.dispose();
+
+			assert(
+				onDisposeSpy.calledOnce,
+				'`onDispose` callback must not be called again.',
+			);
+
+			assert(
+				object.disposed,
+				'Object must be disposed.',
+			);
+		});
+
+		test('executes callback immediately if already disposed', async () => {
+			// this is an abstract class, so we have to create
+			// an anonymous class that extends it
+			const object = new class extends ObservableDisposable { }();
+			disposables.add(object);
+
+			// dispose object and wait for the event to be fired/received
+			object.dispose();
+			await wait(1);
+
+			const onDisposeSpy = spy(() => { });
+			object.onDispose(onDisposeSpy);
+
+			assert(
+				onDisposeSpy.calledOnce,
+				'`onDispose` callback must be called immediately.',
+			);
+
+			await waitRandom(10);
+
+			object.onDispose(onDisposeSpy);
+
+			assert(
+				onDisposeSpy.calledTwice,
+				'`onDispose` callback must be called immediately the second time.',
+			);
+
+			// dispose object and wait for the event to be fired/received
+			object.dispose();
+			await wait(1);
+
+			assert(
+				onDisposeSpy.calledTwice,
+				'`onDispose` callback must not be called again on dispose.',
+			);
+		});
+	});
+
+	suite('asserts', () => {
+		test('not disposed (method)', async () => {
+			// this is an abstract class, so we have to create
+			// an anonymous class that extends it
+			const object: ObservableDisposable = new class extends ObservableDisposable { }();
+			disposables.add(object);
+
+			assert.doesNotThrow(() => {
+				object.assertNotDisposed('Object must not be disposed.');
+			});
+
+			await waitRandom(10);
+
+			assert.doesNotThrow(() => {
+				object.assertNotDisposed('Object must not be disposed.');
+			});
+
+			// dispose object and wait for the event to be fired/received
+			object.dispose();
+			await wait(1);
+
+			assert.throws(() => {
+				object.assertNotDisposed('Object must not be disposed.');
+			});
+
+			await waitRandom(10);
+
+			assert.throws(() => {
+				object.assertNotDisposed('Object must not be disposed.');
+			});
+		});
+
+		test('not disposed (function)', async () => {
+			// this is an abstract class, so we have to create
+			// an anonymous class that extends it
+			const object: ObservableDisposable = new class extends ObservableDisposable { }();
+			disposables.add(object);
+
+			assert.doesNotThrow(() => {
+				assertNotDisposed(
+					object,
+					'Object must not be disposed.',
+				);
+			});
+
+			await waitRandom(10);
+
+			assert.doesNotThrow(() => {
+				assertNotDisposed(
+					object,
+					'Object must not be disposed.',
+				);
+			});
+
+			// dispose object and wait for the event to be fired/received
+			object.dispose();
+			await wait(1);
+
+			assert.throws(() => {
+				assertNotDisposed(
+					object,
+					'Object must not be disposed.',
+				);
+			});
+
+			await waitRandom(10);
+
+			assert.throws(() => {
+				assertNotDisposed(
+					object,
+					'Object must not be disposed.',
+				);
+			});
+		});
+	});
+});

src/vs/editor/common/codecs/markdownCodec/tokens/markdownLink.ts

@@ -4,8 +4,8 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { BaseToken } from '../../baseToken.js';
-import { Range } from '../../../core/range.js';
 import { MarkdownToken } from './markdownToken.js';
+import { IRange, Range } from '../../../core/range.js';
 import { assert } from '../../../../../base/common/assert.js';
 
 /**
@@ -28,7 +28,7 @@ export class MarkdownLink extends MarkdownToken {
 		 */
 		columnNumber: number,
 		/**
-		 * The caprtion of the link, including the square brackets.
+		 * The caption of the link, including the square brackets.
 		 */
 		private readonly caption: string,
 		/**
@@ -105,6 +105,28 @@ export class MarkdownLink extends MarkdownToken {
 		return this.text === other.text;
 	}
 
+	/**
+	 * Get the range of the `link part` of the token.
+	 */
+	public get linkRange(): IRange | undefined {
+		if (this.path.length === 0) {
+			return undefined;
+		}
+
+		const { range } = this;
+
+		// note! '+1' for openning `(` of the link
+		const startColumn = range.startColumn + this.caption.length + 1;
+		const endColumn = startColumn + this.path.length;
+
+		return new Range(
+			range.startLineNumber,
+			startColumn,
+			range.endLineNumber,
+			endColumn,
+		);
+	}
+
 	/**
 	 * Returns a string representation of the token.
 	 */

src/vs/editor/test/common/codecs/linesDecoder.test.ts

@@ -3,16 +3,98 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { TestDecoder } from '../utils/testDecoder.js';
+import assert from 'assert';
 import { Range } from '../../../common/core/range.js';
 import { VSBuffer } from '../../../../base/common/buffer.js';
-import { newWriteableStream } from '../../../../base/common/stream.js';
+import { DisposableStore } from '../../../../base/common/lifecycle.js';
 import { Line } from '../../../common/codecs/linesCodec/tokens/line.js';
+import { TestDecoder, TTokensConsumeMethod } from '../utils/testDecoder.js';
 import { NewLine } from '../../../common/codecs/linesCodec/tokens/newLine.js';
+import { newWriteableStream, WriteableStream } from '../../../../base/common/stream.js';
 import { CarriageReturn } from '../../../common/codecs/linesCodec/tokens/carriageReturn.js';
 import { LinesDecoder, TLineToken } from '../../../common/codecs/linesCodec/linesDecoder.js';
 import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js';
 
+/**
+ * Note! This decoder is also often used to test common logic of abstract {@linkcode BaseDecoder}
+ * class, because the {@linkcode LinesDecoder} is one of the simplest non-abstract decoders we have.
+ */
+suite('LinesDecoder', () => {
+	const disposables = ensureNoDisposablesAreLeakedInTestSuite();
+
+	/**
+	 * Test the core logic with specific method of consuming
+	 * tokens that are produced by a lines decoder instance.
+	 */
+	suite('core logic', () => {
+		testLinesDecoder('async-generator', disposables);
+		testLinesDecoder('consume-all-method', disposables);
+		testLinesDecoder('on-data-event', disposables);
+	});
+
+	suite('settled promise', () => {
+		test('throws if accessed on not-yet-started decoder instance', () => {
+			const test = disposables.add(new TestLinesDecoder());
+
+			assert.throws(
+				() => {
+					// testing the field access that throws here, so
+					// its OK to not use the returned value afterwards
+					// eslint-disable-next-line local/code-no-unused-expressions
+					test.decoder.settled;
+				},
+				[
+					'Cannot get `settled` promise of a stream that has not been started.',
+					'Please call `start()` first.',
+				].join(' '),
+			);
+		});
+	});
+
+	suite('start', () => {
+		test('throws if the decoder object is already `disposed`', () => {
+			const test = disposables.add(new TestLinesDecoder());
+			const { decoder } = test;
+			decoder.dispose();
+
+			assert.throws(
+				decoder.start.bind(decoder),
+				'Cannot start stream that has already disposed.',
+			);
+		});
+
+		test('throws if the decoder object is already `ended`', async () => {
+			const inputStream = newWriteableStream<VSBuffer>(null);
+			const test = disposables.add(new TestLinesDecoder(inputStream));
+			const { decoder } = test;
+
+			setTimeout(() => {
+				test.sendData([
+					'hello',
+					'world :wave:',
+				]);
+			}, 5);
+
+			const receivedTokens = await decoder.start()
+				.consumeAll();
+
+			// a basic sanity check for received tokens
+			assert.strictEqual(
+				receivedTokens.length,
+				3,
+				'Must produce the correct number of tokens.',
+			);
+
+			// validate that calling `start()` after stream has ended throws
+			assert.throws(
+				decoder.start.bind(decoder),
+				'Cannot start stream that has already ended.',
+			);
+		});
+	});
+});
+
+
 /**
  * A reusable test utility that asserts that a `LinesDecoder` instance
  * correctly decodes `inputData` into a stream of `TLineToken` tokens.
@@ -21,7 +103,7 @@ import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/c
  *
  * ```typescript
  * // create a new test utility instance
- * const test = testDisposables.add(new TestLinesDecoder());
+ * const test = disposables.add(new TestLinesDecoder());
  *
  * // run the test
  * await test.run(
@@ -33,108 +115,124 @@ import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/c
  * );
  */
 export class TestLinesDecoder extends TestDecoder<TLineToken, LinesDecoder> {
-	constructor() {
-		const stream = newWriteableStream<VSBuffer>(null);
+	constructor(
+		inputStream?: WriteableStream<VSBuffer>,
+	) {
+		const stream = (inputStream)
+			? inputStream
+			: newWriteableStream<VSBuffer>(null);
+
 		const decoder = new LinesDecoder(stream);
 
 		super(stream, decoder);
 	}
 }
 
-suite('LinesDecoder', () => {
-	const testDisposables = ensureNoDisposablesAreLeakedInTestSuite();
+/**
+ * Common reusable test utility to validate {@linkcode LinesDecoder} logic with
+ * the provided {@linkcode tokensConsumeMethod} way of consuming decoder-produced tokens.
+ *
+ * @throws if a test fails, please see thrown error for failure details.
+ * @param tokensConsumeMethod The way to consume tokens produced by the decoder.
+ * @param disposables Test disposables store.
+ */
+function testLinesDecoder(
+	tokensConsumeMethod: TTokensConsumeMethod,
+	disposables: Pick<DisposableStore, "add">,
+) {
+	suite(tokensConsumeMethod, () => {
+		suite('produces expected tokens', () => {
+			test('input starts with line data', async () => {
+				const test = disposables.add(new TestLinesDecoder());
 
-	suite('produces expected tokens', () => {
-		test('input starts with line data', async () => {
-			const test = testDisposables.add(new TestLinesDecoder());
+				await test.run(
+					' hello world\nhow are you doing?\n\n 😊 \r ',
+					[
+						new Line(1, ' hello world'),
+						new NewLine(new Range(1, 13, 1, 14)),
+						new Line(2, 'how are you doing?'),
+						new NewLine(new Range(2, 19, 2, 20)),
+						new Line(3, ''),
+						new NewLine(new Range(3, 1, 3, 2)),
+						new Line(4, ' 😊 '),
+						new CarriageReturn(new Range(4, 5, 4, 6)),
+						new Line(5, ' '),
+					],
+				);
+			});
 
-			await test.run(
-				' hello world\nhow are you doing?\n\n 😊 \r ',
-				[
-					new Line(1, ' hello world'),
-					new NewLine(new Range(1, 13, 1, 14)),
-					new Line(2, 'how are you doing?'),
-					new NewLine(new Range(2, 19, 2, 20)),
-					new Line(3, ''),
-					new NewLine(new Range(3, 1, 3, 2)),
-					new Line(4, ' 😊 '),
-					new CarriageReturn(new Range(4, 5, 4, 6)),
-					new Line(5, ' '),
-				],
-			);
-		});
+			test('input starts with a new line', async () => {
+				const test = disposables.add(new TestLinesDecoder());
 
-		test('input starts with a new line', async () => {
-			const test = testDisposables.add(new TestLinesDecoder());
+				await test.run(
+					'\nsome text on this line\n\n\nanother 💬 on this line\r\n🤫\n',
+					[
+						new Line(1, ''),
+						new NewLine(new Range(1, 1, 1, 2)),
+						new Line(2, 'some text on this line'),
+						new NewLine(new Range(2, 23, 2, 24)),
+						new Line(3, ''),
+						new NewLine(new Range(3, 1, 3, 2)),
+						new Line(4, ''),
+						new NewLine(new Range(4, 1, 4, 2)),
+						new Line(5, 'another 💬 on this line'),
+						new CarriageReturn(new Range(5, 24, 5, 25)),
+						new NewLine(new Range(5, 25, 5, 26)),
+						new Line(6, '🤫'),
+						new NewLine(new Range(6, 3, 6, 4)),
+					],
+				);
+			});
 
-			await test.run(
-				'\nsome text on this line\n\n\nanother 💬 on this line\r\n🤫\n',
-				[
-					new Line(1, ''),
-					new NewLine(new Range(1, 1, 1, 2)),
-					new Line(2, 'some text on this line'),
-					new NewLine(new Range(2, 23, 2, 24)),
-					new Line(3, ''),
-					new NewLine(new Range(3, 1, 3, 2)),
-					new Line(4, ''),
-					new NewLine(new Range(4, 1, 4, 2)),
-					new Line(5, 'another 💬 on this line'),
-					new CarriageReturn(new Range(5, 24, 5, 25)),
-					new NewLine(new Range(5, 25, 5, 26)),
-					new Line(6, '🤫'),
-					new NewLine(new Range(6, 3, 6, 4)),
-				],
-			);
-		});
+			test('input starts and ends with multiple new lines', async () => {
+				const test = disposables.add(new TestLinesDecoder());
 
-		test('input starts and ends with multiple new lines', async () => {
-			const test = testDisposables.add(new TestLinesDecoder());
+				await test.run(
+					'\n\n\r\nciao! 🗯️\t💭 💥 come\tva?\n\n\n\n\n',
+					[
+						new Line(1, ''),
+						new NewLine(new Range(1, 1, 1, 2)),
+						new Line(2, ''),
+						new NewLine(new Range(2, 1, 2, 2)),
+						new Line(3, ''),
+						new CarriageReturn(new Range(3, 1, 3, 2)),
+						new NewLine(new Range(3, 2, 3, 3)),
+						new Line(4, 'ciao! 🗯️\t💭 💥 come\tva?'),
+						new NewLine(new Range(4, 25, 4, 26)),
+						new Line(5, ''),
+						new NewLine(new Range(5, 1, 5, 2)),
+						new Line(6, ''),
+						new NewLine(new Range(6, 1, 6, 2)),
+						new Line(7, ''),
+						new NewLine(new Range(7, 1, 7, 2)),
+						new Line(8, ''),
+						new NewLine(new Range(8, 1, 8, 2)),
+					],
+				);
+			});
 
-			await test.run(
-				'\n\n\r\nciao! 🗯️\t💭 💥 come\tva?\n\n\n\n\n',
-				[
-					new Line(1, ''),
-					new NewLine(new Range(1, 1, 1, 2)),
-					new Line(2, ''),
-					new NewLine(new Range(2, 1, 2, 2)),
-					new Line(3, ''),
-					new CarriageReturn(new Range(3, 1, 3, 2)),
-					new NewLine(new Range(3, 2, 3, 3)),
-					new Line(4, 'ciao! 🗯️\t💭 💥 come\tva?'),
-					new NewLine(new Range(4, 25, 4, 26)),
-					new Line(5, ''),
-					new NewLine(new Range(5, 1, 5, 2)),
-					new Line(6, ''),
-					new NewLine(new Range(6, 1, 6, 2)),
-					new Line(7, ''),
-					new NewLine(new Range(7, 1, 7, 2)),
-					new Line(8, ''),
-					new NewLine(new Range(8, 1, 8, 2)),
-				],
-			);
-		});
+			test('single carriage return is treated as new line', async () => {
+				const test = disposables.add(new TestLinesDecoder());
 
-		test('single carriage return is treated as new line', async () => {
-			const test = testDisposables.add(new TestLinesDecoder());
-
-			await test.run(
-				'\r\rhaalo! 💥💥 how\'re you?\r ?!\r\n\r\n ',
-				[
-					new Line(1, ''),
-					new CarriageReturn(new Range(1, 1, 1, 2)),
-					new Line(2, ''),
-					new CarriageReturn(new Range(2, 1, 2, 2)),
-					new Line(3, 'haalo! 💥💥 how\'re you?'),
-					new CarriageReturn(new Range(3, 24, 3, 25)),
-					new Line(4, ' ?!'),
-					new CarriageReturn(new Range(4, 4, 4, 5)),
-					new NewLine(new Range(4, 5, 4, 6)),
-					new Line(5, ''),
-					new CarriageReturn(new Range(5, 1, 5, 2)),
-					new NewLine(new Range(5, 2, 5, 3)),
-					new Line(6, ' '),
-				],
-			);
+				await test.run(
+					'\r\rhaalo! 💥💥 how\'re you?\r ?!\r\n\r\n ',
+					[
+						new Line(1, ''),
+						new CarriageReturn(new Range(1, 1, 1, 2)),
+						new Line(2, ''),
+						new CarriageReturn(new Range(2, 1, 2, 2)),
+						new Line(3, 'haalo! 💥💥 how\'re you?'),
+						new CarriageReturn(new Range(3, 24, 3, 25)),
+						new Line(4, ' ?!'),
+						new CarriageReturn(new Range(4, 4, 4, 5)),
+						new NewLine(new Range(4, 5, 4, 6)),
+						new Line(5, ''),
+						new CarriageReturn(new Range(5, 1, 5, 2)),
+						new NewLine(new Range(5, 2, 5, 3)),
+						new Line(6, ' '),
+					],
+				);
+			});
 		});
 	});
-});
+}

src/vs/editor/test/common/utils/testDecoder.ts

@@ -10,9 +10,14 @@ import { BaseToken } from '../../../common/codecs/baseToken.js';
 import { assertDefined } from '../../../../base/common/types.js';
 import { Disposable } from '../../../../base/common/lifecycle.js';
 import { WriteableStream } from '../../../../base/common/stream.js';
-import { randomBoolean } from '../../../../base/test/common/testUtils.js';
 import { BaseDecoder } from '../../../../base/common/codecs/baseDecoder.js';
 
+/**
+ * Kind of decoder tokens consume methods are different ways
+ * consume tokens that a decoder produces out of a byte stream.
+ */
+export type TTokensConsumeMethod = 'async-generator' | 'consume-all-method' | 'on-data-event';
+
 /**
  * A reusable test utility that asserts that the given decoder
  * produces the expected `expectedTokens` sequence of tokens.
@@ -38,7 +43,7 @@ import { BaseDecoder } from '../../../../base/common/codecs/baseDecoder.js';
 export class TestDecoder<T extends BaseToken, D extends BaseDecoder<T>> extends Disposable {
 	constructor(
 		private readonly stream: WriteableStream<VSBuffer>,
-		private readonly decoder: D,
+		public readonly decoder: D,
 	) {
 		super();
 
@@ -46,51 +51,149 @@ export class TestDecoder<T extends BaseToken, D extends BaseDecoder<T>> extends
 	}
 
 	/**
-	 * Run the test sending the `inputData` data to the stream and asserting
-	 * that the decoder produces the `expectedTokens` sequence of tokens.
+	 * Write provided {@linkcode inputData} data to the input byte stream
+	 * asynchronously in the background in small random-length chunks.
+	 *
+	 * @param inputData Input data to send.
 	 */
-	public async run(
+	public sendData(
 		inputData: string | string[],
-		expectedTokens: readonly T[],
-	): Promise<void> {
+	): this {
 		// if input data was passed as an array of lines,
 		// join them into a single string with newlines
 		if (Array.isArray(inputData)) {
 			inputData = inputData.join('\n');
 		}
 
-		// write the data to the stream after a short delay to ensure
-		// that the the data is sent after the reading loop below
-		setTimeout(() => {
-			let inputDataBytes = VSBuffer.fromString(inputData);
+		// write the input data to the stream in multiple random-length
+		// chunks to simulate real input stream data flows
+		let inputDataBytes = VSBuffer.fromString(inputData);
+		const interval = setInterval(() => {
+			if (inputDataBytes.byteLength <= 0) {
+				clearInterval(interval);
+				this.stream.end();
 
-			// write the input data to the stream in multiple random-length chunks
-			while (inputDataBytes.byteLength > 0) {
-				const dataToSend = inputDataBytes.slice(0, randomInt(inputDataBytes.byteLength));
-				this.stream.write(dataToSend);
-				inputDataBytes = inputDataBytes.slice(dataToSend.byteLength);
+				return;
 			}
 
-			this.stream.end();
-		}, 25);
+			const dataToSend = inputDataBytes.slice(0, randomInt(inputDataBytes.byteLength));
+			this.stream.write(dataToSend);
+			inputDataBytes = inputDataBytes.slice(dataToSend.byteLength);
+		}, randomInt(5));
+
+		return this;
+	}
+
+	/**
+	 * Run the test sending the `inputData` data to the stream and asserting
+	 * that the decoder produces the `expectedTokens` sequence of tokens.
+	 *
+	 * @param inputData Input data of the input byte stream.
+	 * @param expectedTokens List of expected tokens the test token must produce.
+	 * @param tokensConsumeMethod *Optional* method of consuming the decoder stream.
+	 *       					  Defaults to a random method (see {@linkcode randomTokensConsumeMethod}).
+	 */
+	public async run(
+		inputData: string | string[],
+		expectedTokens: readonly T[],
+		tokensConsumeMethod: TTokensConsumeMethod = this.randomTokensConsumeMethod(),
+	): Promise<void> {
+		try {
+			// initiate the data sending flow
+			this.sendData(inputData);
+
+			// consume the decoder tokens based on specified
+			// (or randomly generated) tokens consume method
+			const receivedTokens: T[] = [];
+			switch (tokensConsumeMethod) {
+				// test the `async iterator` code path
+				case 'async-generator': {
+					for await (const token of this.decoder) {
+						if (token === null) {
+							break;
+						}
+
+						receivedTokens.push(token);
+					}
 
-		// randomly use either the `async iterator` or the `.consume()`
-		// variants of getting tokens, they both must yield equal results
-		const receivedTokens: T[] = [];
-		if (randomBoolean()) {
-			// test the `async iterator` code path
-			for await (const token of this.decoder) {
-				if (token === null) {
 					break;
 				}
+				// test the `.consumeAll()` method code path
+				case 'consume-all-method': {
+					receivedTokens.push(...(await this.decoder.consumeAll()));
+					break;
+				}
+				// test the `.onData()` event consume flow
+				case 'on-data-event': {
+					this.decoder.onData((token) => {
+						receivedTokens.push(token);
+					});
 
-				receivedTokens.push(token);
+					// in this case we also test the `settled` promise of the decoder
+					await this.decoder.settled;
+
+					break;
+				}
+				// ensure that the switch block is exhaustive
+				default: {
+					throw new Error(`Unknown consume method '${tokensConsumeMethod}'.`);
+				}
 			}
-		} else {
-			// test the `.consume()` code path
-			receivedTokens.push(...(await this.decoder.consumeAll()));
-		}
 
+			// validate the received tokens
+			this.validateReceivedTokens(
+				receivedTokens,
+				expectedTokens,
+			);
+		} catch (error) {
+			assertDefined(
+				error,
+				`An non-nullable error must be thrown.`,
+			);
+			assert(
+				error instanceof Error,
+				`An error error instance must be thrown.`,
+			);
+
+			// add the tokens consume method to the error message so we
+			// would know which method of consuming the tokens failed exactly
+			error.message = `[${tokensConsumeMethod}] ${error.message}`;
+		}
+	}
+
+	/**
+	 * Randomly generate a tokens consume method type for the test.
+	 */
+	private randomTokensConsumeMethod(): TTokensConsumeMethod {
+		const testConsumeMethodIndex = randomInt(2);
+
+		switch (testConsumeMethodIndex) {
+			// test the `async iterator` code path
+			case 0: {
+				return 'async-generator';
+			}
+			// test the `.consumeAll()` method code path
+			case 1: {
+				return 'consume-all-method';
+			}
+			// test the `.onData()` event consume flow
+			case 2: {
+				return 'on-data-event';
+			}
+			// ensure that the switch block is exhaustive
+			default: {
+				throw new Error(`Unknown consume method index '${testConsumeMethodIndex}'.`);
+			}
+		}
+	}
+
+	/**
+	 * Validate that received tokens list is equal to the expected one.
+	 */
+	private validateReceivedTokens(
+		receivedTokens: readonly T[],
+		expectedTokens: readonly T[],
+	) {
 		for (let i = 0; i < expectedTokens.length; i++) {
 			const expectedToken = expectedTokens[i];
 			const receivedtoken = receivedTokens[i];

src/vs/workbench/contrib/chat/browser/attachments/instructionsAttachment/instructionsAttachment.ts

@@ -97,7 +97,7 @@ export class InstructionsAttachmentWidget extends Disposable {
 		this.renderDisposables.clear();
 		this.domNode.classList.remove('warning', 'error', 'disabled');
 
-		const { enabled, resolveIssue: errorCondition } = this.model;
+		const { enabled, topError } = this.model;
 		if (!enabled) {
 			this.domNode.classList.add('disabled');
 		}
@@ -120,11 +120,15 @@ export class InstructionsAttachmentWidget extends Disposable {
 		// if there are some errors/warning during the process of resolving
 		// attachment references (including all the nested child references),
 		// add the issue details in the hover title for the attachment
-		if (errorCondition) {
-			const { type, message: details } = errorCondition;
-			this.domNode.classList.add(type);
+		if (topError) {
+			const { isRootError, message: details } = topError;
+			const isWarning = !isRootError;
 
-			const errorCaption = type === 'warning'
+			this.domNode.classList.add(
+				(isWarning) ? 'error' : 'warning',
+			);
+
+			const errorCaption = (isWarning)
 				? localize('warning', "Warning")
 				: localize('error', "Error");
 

src/vs/workbench/contrib/chat/browser/chat.contribution.ts

@@ -81,6 +81,7 @@ import { ChatRelatedFilesContribution } from './contrib/chatInputRelatedFilesCon
 import { ChatQuotasService, ChatQuotasStatusBarEntry, IChatQuotasService } from './chatQuotasService.js';
 import { BuiltinToolsContribution } from './tools/tools.js';
 import { ChatSetupContribution } from './chatSetup.js';
+import '../common/promptSyntax/languageFeatures/promptLinkProvider.js';
 
 // Register configuration
 const configurationRegistry = Registry.as<IConfigurationRegistry>(ConfigurationExtensions.Configuration);

src/vs/workbench/contrib/chat/browser/chatAttachmentModel/chatInstructionsAttachment.ts

@@ -3,43 +3,11 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { localize } from '../../../../../nls.js';
 import { URI } from '../../../../../base/common/uri.js';
 import { Emitter } from '../../../../../base/common/event.js';
-import { basename } from '../../../../../base/common/resources.js';
-import { assertDefined } from '../../../../../base/common/types.js';
 import { Disposable } from '../../../../../base/common/lifecycle.js';
 import { FilePromptParser } from '../../common/promptSyntax/parsers/filePromptParser.js';
 import { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';
-import { FailedToResolveContentsStream, FileOpenFailed, NonPromptSnippetFile, ParseError, RecursiveReference } from '../../common/promptFileReferenceErrors.js';
-
-/**
- * Well-known localized error messages.
- */
-const errorMessages = {
-	recursion: localize('chatPromptInstructionsRecursiveReference', 'Recursive reference found'),
-	fileOpenFailed: localize('chatPromptInstructionsFileOpenFailed', 'Failed to open file'),
-	brokenChild: localize('chatPromptInstructionsBrokenReference', 'Contains a broken reference that will be ignored'),
-};
-
-/**
- * Object that represents an error that may occur during
- * the process of resolving prompt instructions reference.
- */
-interface IIssue {
-	/**
-	 * Type of the failure. Currently all errors that occur on
-	 * the "main" root reference directly attached to the chat
-	 * are considered to be `error`s, while all failures on nested
-	 * child references are considered to be `warning`s.
-	 */
-	type: 'error' | 'warning';
-
-	/**
-	 * Error or warning message.
-	 */
-	message: string;
-}
 
 /**
  * Model for a single chat prompt instructions attachment.
@@ -62,15 +30,12 @@ export class ChatInstructionsAttachmentModel extends Disposable {
 	 * child references it may contain.
 	 */
 	public get references(): readonly URI[] {
-		const { reference, enabled, resolveIssue } = this;
+		const { reference, enabled } = this;
+		const { errorCondition } = this.reference;
 
 		// return no references if the attachment is disabled
-		if (!enabled) {
-			return [];
-		}
-
-		// if the model has an error, return no references
-		if (resolveIssue && !(resolveIssue instanceof NonPromptSnippetFile)) {
+		// or if this object itself has an error
+		if (!enabled || errorCondition) {
 			return [];
 		}
 
@@ -82,120 +47,12 @@ export class ChatInstructionsAttachmentModel extends Disposable {
 		];
 	}
 
-
 	/**
-	 * If the prompt instructions reference (or any of its child references) has
-	 * failed to resolve, this field contains the failure details, otherwise `undefined`.
-	 *
-	 * See {@linkcode IIssue}.
+	 * Get the top-level error of the prompt instructions
+	 * reference, if any.
 	 */
-	public get resolveIssue(): IIssue | undefined {
-		const { errorCondition } = this._reference;
-
-		const errorConditions = this.collectErrorConditions();
-		if (errorConditions.length === 0) {
-			return undefined;
-		}
-
-		const [firstError, ...restErrors] = errorConditions;
-
-		// if the first error is the error of the root reference,
-		// then return it as an `error` otherwise use `warning`
-		const isRootError = (firstError === errorCondition);
-		const type = (isRootError)
-			? 'error'
-			: 'warning';
-
-		const moreSuffix = restErrors.length > 0
-			? `\n-\n +${restErrors.length} more error${restErrors.length > 1 ? 's' : ''}`
-			: '';
-
-		const errorMessage = this.getErrorMessage(firstError, isRootError);
-		return {
-			type,
-			message: `${errorMessage}${moreSuffix}`,
-		};
-	}
-
-	/**
-	 * Get message for the provided error condition object.
-	 *
-	 * @param error Error object.
-	 * @param isRootError If the error happened on the the "main" root reference.
-	 * @returns Error message.
-	 */
-	private getErrorMessage(
-		error: ParseError,
-		isRootError: boolean,
-	): string {
-		// if a child error - the error is somewhere in the nested references tree,
-		// then use message prefix to highlight that this is not a root error
-		const prefix = (!isRootError)
-			? `${errorMessages.brokenChild}: `
-			: '';
-
-		// if failed to open a file, return approprivate message and the file path
-		if (error instanceof FileOpenFailed || error instanceof FailedToResolveContentsStream) {
-			return `${prefix}${errorMessages.fileOpenFailed} '${error.uri.path}'.`;
-		}
-
-		// if a recursion, provide the entire recursion path so users can use
-		// it for the debugging purposes
-		if (error instanceof RecursiveReference) {
-			const { recursivePath } = error;
-
-			const recursivePathString = recursivePath
-				.map((path) => {
-					return basename(URI.file(path));
-				})
-				.join(' -> ');
-
-			return `${prefix}${errorMessages.recursion}:\n${recursivePathString}`;
-		}
-
-		return `${prefix}${error.message}`;
-	}
-
-	/**
-	 * Collect all failures that may have occurred during the process of resolving
-	 * references in the entire references tree, including the current root reference.
-	 *
-	 * @returns List of errors in the references tree.
-	 */
-	private collectErrorConditions(): ParseError[] {
-		const result: ParseError[] = [];
-
-		// add error conditions of this object
-		if (this._reference.errorCondition) {
-			result.push(this._reference.errorCondition);
-		}
-
-		// collect error conditions of all child references
-		const childErrorConditions = this.reference
-			// get entire reference tree
-			.allReferences
-			// filter out children without error conditions or
-			// the ones that are non-prompt snippet files
-			.filter((childReference) => {
-				const { errorCondition } = childReference;
-
-				return errorCondition && !(errorCondition instanceof NonPromptSnippetFile);
-			})
-			// map to error condition objects
-			.map((childReference): ParseError => {
-				const { errorCondition } = childReference;
-
-				// `must` always be `true` because of the `filter` call above
-				assertDefined(
-					errorCondition,
-					`Error condition must be present for '${childReference.uri.path}'.`,
-				);
-
-				return errorCondition;
-			});
-		result.push(...childErrorConditions);
-
-		return result;
+	public get topError() {
+		return this.reference.topError;
 	}
 
 	/**

src/vs/workbench/contrib/chat/browser/media/chat.css

@@ -960,7 +960,7 @@ have to be updated for changes to the rules above, or to support more deeply nes
 	opacity: 0.75;
 }
 /*
- * This overly specific CSS selector is needed to beat priority of some
+ * This overly-specific CSS selector is needed to beat priority of some
  * styles applied on the the `.chat-attached-context-attachment` element.
  */
 .chat-attached-context .chat-prompt-instructions-attachments .chat-prompt-instructions-attachment.error.implicit,

src/vs/workbench/contrib/chat/common/promptFileReferenceErrors.ts

@@ -49,10 +49,9 @@ export class FailedToResolveContentsStream extends ParseError {
 	constructor(
 		public readonly uri: URI,
 		public readonly originalError: unknown,
+		message: string = `Failed to resolve prompt contents stream for '${uri.toString()}': ${originalError}.`,
 	) {
-		super(
-			`Failed to resolve prompt contents stream for '${uri.toString()}': ${originalError}.`,
-		);
+		super(message);
 	}
 }
 
@@ -75,15 +74,16 @@ export abstract class ResolveError extends ParseError {
 /**
  * Error that reflects the case when attempt to open target file fails.
  */
-export class FileOpenFailed extends ResolveError {
+export class FileOpenFailed extends FailedToResolveContentsStream {
 	public override errorType = 'FileOpenError';
 
 	constructor(
 		uri: URI,
-		public readonly originalError: unknown,
+		originalError: unknown,
 	) {
 		super(
 			uri,
+			originalError,
 			`Failed to open file '${uri.toString()}': ${originalError}.`,
 		);
 	}

src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { assert } from '../../../../../../../base/common/assert.js';
-import { Range } from '../../../../../../../editor/common/core/range.js';
+import { IRange, Range } from '../../../../../../../editor/common/core/range.js';
 import { BaseToken } from '../../../../../../../editor/common/codecs/baseToken.js';
 import { Word } from '../../../../../../../editor/common/codecs/simpleCodec/tokens/word.js';
 
@@ -93,6 +93,24 @@ export class FileReference extends BaseToken {
 		return this.text === other.text;
 	}
 
+	/**
+	 * Get the range of the `link part` of the token (e.g.,
+	 * the `/path/to/file.md` part of `#file:/path/to/file.md`).
+	 */
+	public get linkRange(): IRange | undefined {
+		if (this.path.length === 0) {
+			return undefined;
+		}
+
+		const { range } = this;
+		return new Range(
+			range.startLineNumber,
+			range.startColumn + TOKEN_START.length,
+			range.endLineNumber,
+			range.endColumn,
+		);
+	}
+
 	/**
 	 * Return a string representation of the token.
 	 */

src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts

@@ -5,6 +5,7 @@
 
 import { IPromptContentsProvider } from './types.js';
 import { URI } from '../../../../../../base/common/uri.js';
+import { assert } from '../../../../../../base/common/assert.js';
 import { assertDefined } from '../../../../../../base/common/types.js';
 import { CancellationError } from '../../../../../../base/common/errors.js';
 import { PromptContentsProviderBase } from './promptContentsProviderBase.js';
@@ -55,9 +56,10 @@ export class FilePromptContentProvider extends PromptContentsProviderBase<FileCh
 		_event: FileChangesEvent | 'full',
 		cancellationToken?: CancellationToken,
 	): Promise<VSBufferReadableStream> {
-		if (cancellationToken?.isCancellationRequested) {
-			throw new CancellationError();
-		}
+		assert(
+			!cancellationToken?.isCancellationRequested,
+			new CancellationError(),
+		);
 
 		// get the binary stream of the file contents
 		let fileStream;

src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts

@@ -10,7 +10,7 @@ import { assert } from '../../../../../../base/common/assert.js';
 import { CancellationError } from '../../../../../../base/common/errors.js';
 import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js';
 import { CancellationToken } from '../../../../../../base/common/cancellation.js';
-import { TrackedDisposable } from '../../../../../../base/common/trackedDisposable.js';
+import { ObservableDisposable } from '../../../../../../base/common/observableDisposable.js';
 import { FailedToResolveContentsStream, ParseError } from '../../promptFileReferenceErrors.js';
 import { cancelPreviousCalls } from '../../../../../../base/common/decorators/cancelPreviousCalls.js';
 
@@ -35,7 +35,7 @@ export const PROMP_SNIPPET_FILE_EXTENSION: string = '.prompt.md';
  */
 export abstract class PromptContentsProviderBase<
 	TChangeEvent extends NonNullable<unknown>,
-> extends TrackedDisposable implements IPromptContentsProvider {
+> extends ObservableDisposable implements IPromptContentsProvider {
 	/**
 	 * Internal event emitter for the prompt contents change event. Classes that extend
 	 * this abstract class are responsible to use this emitter to fire the contents change

src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts

@@ -49,29 +49,44 @@ export class TextModelContentsProvider extends PromptContentsProviderBase<IModel
 
 		// provide the changed lines to the stream incrementaly and asynchronously
 		// to avoid blocking the main thread and save system resources used
-		let i = 0;
+		let i = 1;
 		const interval = setInterval(() => {
-			if (this.model.isDisposed() || cancellationToken?.isCancellationRequested) {
-				clearInterval(interval);
-				stream.error(new CancellationError());
-				stream.end();
-				return;
-			}
-
-			// write the current line to the stream
-			stream.write(
-				VSBuffer.fromString(this.model.getLineContent(i)),
-			);
-
-			// use the next line in the next iteration
-			i++;
-
-			// if we have written all lines, end the stream and stop
-			// the interval timer
+			// if we have written all lines or lines count is zero,
+			// end the stream and stop the interval timer
 			if (i >= linesCount) {
 				clearInterval(interval);
 				stream.end();
+				stream.destroy();
 			}
+
+			// if model was disposed or cancellation was requested,
+			// end the stream with an error and stop the interval timer
+			if (this.model.isDisposed() || cancellationToken?.isCancellationRequested) {
+				clearInterval(interval);
+				stream.error(new CancellationError());
+				stream.destroy();
+				return;
+			}
+
+			try {
+				// write the current line to the stream
+				stream.write(
+					VSBuffer.fromString(this.model.getLineContent(i)),
+				);
+
+				// for all lines exept the last one, write the EOL character
+				// to separate the lines in the stream
+				if (i !== linesCount) {
+					stream.write(
+						VSBuffer.fromString(this.model.getEOL()),
+					);
+				}
+			} catch (error) {
+				console.log(this.uri, i, error);
+			}
+
+			// use the next line in the next iteration
+			i++;
 		}, 1);
 
 		return stream;

src/vs/workbench/contrib/chat/common/promptSyntax/languageFeatures/promptLinkProvider.ts

@@ -0,0 +1,135 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { assert } from '../../../../../../base/common/assert.js';
+import { ITextModel } from '../../../../../../editor/common/model.js';
+import { assertDefined } from '../../../../../../base/common/types.js';
+import { Disposable } from '../../../../../../base/common/lifecycle.js';
+import { NonPromptSnippetFile } from '../../promptFileReferenceErrors.js';
+import { ObjectCache } from '../../../../../../base/common/objectCache.js';
+import { CancellationError } from '../../../../../../base/common/errors.js';
+import { TextModelPromptParser } from '../parsers/textModelPromptParser.js';
+import { CancellationToken } from '../../../../../../base/common/cancellation.js';
+import { Registry } from '../../../../../../platform/registry/common/platform.js';
+import { LifecyclePhase } from '../../../../../services/lifecycle/common/lifecycle.js';
+import { ILink, ILinksList, LinkProvider } from '../../../../../../editor/common/languages.js';
+import { PROMP_SNIPPET_FILE_EXTENSION } from '../contentProviders/promptContentsProviderBase.js';
+import { ILanguageFeaturesService } from '../../../../../../editor/common/services/languageFeatures.js';
+import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js';
+import { IWorkbenchContributionsRegistry, Extensions as WorkbenchExtensions } from '../../../../../common/contributions.js';
+
+/**
+ * Prompt files language selector.
+ */
+const languageSelector = {
+	pattern: `**/*${PROMP_SNIPPET_FILE_EXTENSION}`,
+};
+
+/**
+ * Provides link references for prompt files.
+ */
+export class PromptLinkProvider extends Disposable implements LinkProvider {
+	/**
+	 * Cache of text model content prompt parsers.
+	 */
+	private readonly parserProvider: ObjectCache<TextModelPromptParser, ITextModel>;
+
+	constructor(
+		@IInstantiationService private readonly initService: IInstantiationService,
+		@ILanguageFeaturesService private readonly languageService: ILanguageFeaturesService,
+	) {
+		super();
+
+		this.languageService.linkProvider.register(languageSelector, this);
+		this.parserProvider = this._register(new ObjectCache(this.createParser.bind(this)));
+	}
+
+	/**
+	 * Create new prompt parser instance for the provided text model.
+	 *
+	 * @param model - text model to create the parser for
+	 * @param initService - the instantiation service
+	 */
+	private createParser(
+		model: ITextModel,
+	): TextModelPromptParser & { disposed: false } {
+		const parser: TextModelPromptParser = this.initService.createInstance(
+			TextModelPromptParser,
+			model,
+			[],
+		);
+
+		parser.assertNotDisposed(
+			'Created prompt parser must not be disposed.',
+		);
+
+		return parser;
+	}
+
+	/**
+	 * Provide list of links for the provided text model.
+	 */
+	public async provideLinks(
+		model: ITextModel,
+		token: CancellationToken,
+	): Promise<ILinksList> {
+		assert(
+			!token.isCancellationRequested,
+			new CancellationError(),
+		);
+
+		const parser = this.parserProvider.get(model);
+		assert(
+			!parser.disposed,
+			'Prompt parser must not be disposed.',
+		);
+
+		// start the parser in case it was not started yet,
+		// and wait for it to settle to a final result
+		const { references } = await parser
+			.start()
+			.settled();
+
+		// validate that the cancellation was not yet requested
+		assert(
+			!token.isCancellationRequested,
+			new CancellationError(),
+		);
+
+		// filter out references that are not valid links
+		const links: ILink[] = references
+			.filter((reference) => {
+				const { errorCondition, linkRange } = reference;
+				if (!errorCondition && linkRange) {
+					return true;
+				}
+
+				return errorCondition instanceof NonPromptSnippetFile;
+			})
+			.map((reference) => {
+				const { linkRange } = reference;
+
+				// must always be true because of the filter above
+				assertDefined(
+					linkRange,
+					'Link range must be defined.',
+				);
+
+
+				return {
+					range: linkRange,
+					url: reference.uri,
+				};
+			});
+
+		return {
+			links,
+		};
+	}
+}
+
+// register the text model prompt decorators provider as a workbench contribution
+Registry.as<IWorkbenchContributionsRegistry>(WorkbenchExtensions.Workbench)
+	.registerWorkbenchContribution(PromptLinkProvider, LifecyclePhase.Eventually);

src/vs/workbench/contrib/chat/common/promptSyntax/languageFeatures/types.d.ts

@@ -0,0 +1,37 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { IRange } from '../../../../../../editor/common/core/range.js';
+import { ModelDecorationOptions } from '../../../../../../editor/common/model/textModel.js';
+
+/**
+ * Decoration object.
+ */
+export interface ITextModelDecoration {
+	/**
+	 * Range of the decoration.
+	 */
+	range: IRange;
+
+	/**
+	 * Associated decoration options.
+	 */
+	options: ModelDecorationOptions;
+}
+
+/**
+ * Decoration CSS class names.
+ */
+export enum DecorationClassNames {
+	/**
+	 * CSS class name for `default` prompt syntax decoration.
+	 */
+	default = 'prompt-decoration',
+
+	/**
+	 * CSS class name for `file reference` prompt syntax decoration.
+	 */
+	fileReference = DecorationClassNames.default,
+}

src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts

@@ -3,32 +3,44 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { IPromptFileReference } from './types.js';
+import { localize } from '../../../../../../nls.js';
 import { URI } from '../../../../../../base/common/uri.js';
 import { ChatPromptCodec } from '../codecs/chatPromptCodec.js';
 import { Emitter } from '../../../../../../base/common/event.js';
+import { assert } from '../../../../../../base/common/assert.js';
+import { IPromptFileReference, IResolveError } from './types.js';
 import { FileReference } from '../codecs/tokens/fileReference.js';
+import { ChatPromptDecoder } from '../codecs/chatPromptDecoder.js';
+import { IRange } from '../../../../../../editor/common/core/range.js';
+import { assertDefined } from '../../../../../../base/common/types.js';
 import { IPromptContentsProvider } from '../contentProviders/types.js';
+import { DeferredPromise } from '../../../../../../base/common/async.js';
 import { ILogService } from '../../../../../../platform/log/common/log.js';
 import { basename, extUri } from '../../../../../../base/common/resources.js';
 import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js';
-import { TrackedDisposable } from '../../../../../../base/common/trackedDisposable.js';
+import { ObservableDisposable } from '../../../../../../base/common/observableDisposable.js';
 import { FilePromptContentProvider } from '../contentProviders/filePromptContentsProvider.js';
+import { PROMP_SNIPPET_FILE_EXTENSION } from '../contentProviders/promptContentsProviderBase.js';
 import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js';
 import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js';
 import { MarkdownLink } from '../../../../../../editor/common/codecs/markdownCodec/tokens/markdownLink.js';
-import { FileOpenFailed, NonPromptSnippetFile, RecursiveReference, ParseError } from '../../promptFileReferenceErrors.js';
+import { FileOpenFailed, NonPromptSnippetFile, RecursiveReference, ParseError, FailedToResolveContentsStream } from '../../promptFileReferenceErrors.js';
+
+/**
+ * Well-known localized error messages.
+ */
+const errorMessages = {
+	recursion: localize('chatPromptInstructionsRecursiveReference', 'Recursive reference found'),
+	fileOpenFailed: localize('chatPromptInstructionsFileOpenFailed', 'Failed to open file'),
+	streamOpenFailed: localize('chatPromptInstructionsStreamOpenFailed', 'Failed to open contents stream'),
+	brokenChild: localize('chatPromptInstructionsBrokenReference', 'Contains a broken reference that will be ignored'),
+};
 
 /**
  * Error conditions that may happen during the file reference resolution.
  */
 export type TErrorCondition = FileOpenFailed | RecursiveReference | NonPromptSnippetFile;
 
-/**
- * File extension for the prompt snippets.
- */
-export const PROMP_SNIPPET_FILE_EXTENSION: string = '.prompt.md';
-
 /**
  * Configuration key for the prompt snippets feature.
  */
@@ -38,8 +50,7 @@ const PROMPT_SNIPPETS_CONFIG_KEY: string = 'chat.experimental.prompt-snippets';
  * Base prompt parser class that provides a common interface for all
  * prompt parsers that are responsible for parsing chat prompt syntax.
  */
-export abstract class BasePromptParser<T extends IPromptContentsProvider> extends TrackedDisposable {
-
+export abstract class BasePromptParser<T extends IPromptContentsProvider> extends ObservableDisposable {
 	/**
 	 * List of file references in the current branch of the file reference tree.
 	 */
@@ -70,23 +81,71 @@ export abstract class BasePromptParser<T extends IPromptContentsProvider> extend
 		return this._errorCondition;
 	}
 
-	/**
-	 * Whether file reference resolution was attempted at least once.
-	 */
-	private _resolveAttempted: boolean = false;
-
 	/**
 	 * Whether file references resolution failed.
 	 * Set to `undefined` if the `resolve` method hasn't been ever called yet.
 	 */
 	public get resolveFailed(): boolean | undefined {
-		if (!this._resolveAttempted) {
+		if (!this.firstParseResult.gotFirstResult) {
 			return undefined;
 		}
 
 		return !!this._errorCondition;
 	}
 
+	/**
+	 * The promise is resolved when at least one parse result (a stream or
+	 * an error) has been received from the prompt contents provider.
+	 */
+	private firstParseResult = new FirstParseResult();
+
+	/**
+	 * Returned promise is resolved when the parser process is settled.
+	 * The settled state means that the prompt parser stream exists and
+	 * has ended, or an error condition has been set in case of failure.
+	 *
+	 * Furthermore, this function can be called multiple times and will
+	 * block until the latest prompt contents parsing logic is settled
+	 * (e.g., for every `onContentChanged` event of the prompt source).
+	 */
+	public async settled(): Promise<this> {
+		assert(
+			this.started,
+			'Cannot wait on the parser that did not start yet.',
+		);
+
+		await this.firstParseResult.promise;
+
+		if (this.errorCondition) {
+			return this;
+		}
+
+		assertDefined(
+			this.stream,
+			'No stream reference found.',
+		);
+
+		await this.stream.settled;
+
+		return this;
+	}
+
+	/**
+	 * Same as {@linkcode settled} but also waits for all possible
+	 * nested child prompt references and their children to be settled.
+	 */
+	public async settledAll(): Promise<this> {
+		await this.settled();
+
+		await Promise.allSettled(
+			this.references.map((reference) => {
+				return reference.settledAll();
+			}),
+		);
+
+		return this;
+	}
+
 	constructor(
 		private readonly promptContentsProvider: T,
 		seenReferences: string[] = [],
@@ -106,8 +165,8 @@ export abstract class BasePromptParser<T extends IPromptContentsProvider> extend
 			seenReferences.push(this.uri.path);
 
 			this._errorCondition = new RecursiveReference(this.uri, seenReferences);
-			this._resolveAttempted = true;
 			this._onUpdate.fire();
+			this.firstParseResult.complete();
 
 			return this;
 		}
@@ -117,24 +176,22 @@ export abstract class BasePromptParser<T extends IPromptContentsProvider> extend
 		// even if the file doesn't exist, we would never end up in the recursion
 		seenReferences.push(this.uri.path);
 
-		let currentStream: VSBufferReadableStream | undefined;
 		this._register(
 			this.promptContentsProvider.onContentChanged((streamOrError) => {
-				// destroy previously received stream
-				currentStream?.destroy();
-
-				if (!(streamOrError instanceof ParseError)) {
-					// save the current stream object so it can be destroyed when/if
-					// a new stream is received
-					currentStream = streamOrError;
-				}
-
 				// process the the received message
 				this.onContentsChanged(streamOrError, seenReferences);
+
+				// indicate that we've received at least one `onContentChanged` event
+				this.firstParseResult.complete();
 			}),
 		);
 	}
 
+	/**
+	 * The latest received stream of prompt tokens, if any.
+	 */
+	private stream: ChatPromptDecoder | undefined;
+
 	/**
 	 * Handler the event event that is triggered when prompt contents change.
 	 *
@@ -149,11 +206,11 @@ export abstract class BasePromptParser<T extends IPromptContentsProvider> extend
 		streamOrError: VSBufferReadableStream | ParseError,
 		seenReferences: string[],
 	): void {
-		// set the flag indicating that reference resolution was attempted
-		this._resolveAttempted = true;
-
-		// prefix for all log messages produced by this callback
-		const logPrefix = `[prompt parser][${basename(this.uri)}]`;
+		// dispose and cleanup the previously received stream
+		// object or an error condition, if any received yet
+		this.stream?.dispose();
+		delete this.stream;
+		delete this._errorCondition;
 
 		// dispose all currently existing references
 		this.disposeReferences();
@@ -166,24 +223,15 @@ export abstract class BasePromptParser<T extends IPromptContentsProvider> extend
 			return;
 		}
 
-		// cleanup existing error condition (if any)
-		delete this._errorCondition;
-
 		// decode the byte stream to a stream of prompt tokens
-		const stream = ChatPromptCodec.decode(streamOrError);
+		this.stream = ChatPromptCodec.decode(streamOrError);
 
-		// on error or stream end, dispose the stream
-		stream.on('error', (error) => {
-			stream.dispose();
-
-			this.logService.warn(
-				`${logPrefix} received an error on the chat prompt decoder stream: ${error}`,
-			);
-		});
-		stream.on('end', stream.dispose.bind(stream));
+		// on error or stream end, dispose the stream and fire the update event
+		this.stream.on('error', this.onStreamEnd.bind(this, this.stream));
+		this.stream.on('end', this.onStreamEnd.bind(this, this.stream));
 
 		// when some tokens received, process and store the references
-		stream.on('data', (token) => {
+		this.stream.on('data', (token) => {
 			if (token instanceof FileReference) {
 				this.onReference(token, [...seenReferences]);
 			}
@@ -196,16 +244,16 @@ export abstract class BasePromptParser<T extends IPromptContentsProvider> extend
 		});
 
 		// calling `start` on a disposed stream throws, so we warn and return instead
-		if (stream.disposed) {
+		if (this.stream.disposed) {
 			this.logService.warn(
-				`${logPrefix} cannot start stream that has been already disposed, aborting`,
+				`[prompt parser][${basename(this.uri)}] cannot start stream that has been already disposed, aborting`,
 			);
 
 			return;
 		}
 
 		// start receiving data on the stream
-		stream.start();
+		this.stream.start();
 	}
 
 	/**
@@ -228,6 +276,25 @@ export abstract class BasePromptParser<T extends IPromptContentsProvider> extend
 		return this;
 	}
 
+	/**
+	 * Handle the `stream` end event.
+	 *
+	 * @param stream The stream that has ended.
+	 * @param error Optional error object if stream ended with an error.
+	 */
+	private onStreamEnd(
+		_stream: ChatPromptDecoder,
+		error?: Error,
+	): this {
+		this.logService.warn(
+			`[prompt parser][${basename(this.uri)}]} received an error on the chat prompt decoder stream: ${error}`,
+		);
+
+		this._onUpdate.fire();
+
+		return this;
+	}
+
 	/**
 	 * Dispose all currently held references.
 	 */
@@ -239,17 +306,30 @@ export abstract class BasePromptParser<T extends IPromptContentsProvider> extend
 		this._references.length = 0;
 	}
 
+	/**
+	 * Private attribute to track if the {@linkcode start}
+	 * method has been already called at least once.
+	 */
+	private started: boolean = false;
+
 	/**
 	 * Start the prompt parser.
 	 */
 	public start(): this {
-		// if already in error state, nothing to do
+		// if already started, nothing to do
+		if (this.started) {
+			return this;
+		}
+		this.started = true;
+
+
+		// if already in the error state that could be set
+		// in the constructor, then nothing to do
 		if (this.errorCondition) {
 			return this;
 		}
 
 		this.promptContentsProvider.start();
-
 		return this;
 	}
 
@@ -296,7 +376,7 @@ export abstract class BasePromptParser<T extends IPromptContentsProvider> extend
 
 	/**
 	 * Get a list of all references of the prompt, including
-	 * all possible nested references its children may contain.
+	 * all possible nested references its children may have.
 	 */
 	public get allReferences(): readonly IPromptFileReference[] {
 		const result: IPromptFileReference[] = [];
@@ -331,6 +411,111 @@ export abstract class BasePromptParser<T extends IPromptContentsProvider> extend
 			.map(child => child.uri);
 	}
 
+	/**
+	 * List of all errors that occurred while resolving the current
+	 * reference including all possible errors of nested children.
+	 */
+	public get allErrors(): ParseError[] {
+		const result: ParseError[] = [];
+
+		// collect error conditions of all child references
+		const childErrorConditions = this
+			// get entire reference tree
+			.allReferences
+			// filter out children without error conditions or
+			// the ones that are non-prompt snippet files
+			.filter((childReference) => {
+				const { errorCondition } = childReference;
+
+				return errorCondition && !(errorCondition instanceof NonPromptSnippetFile);
+			})
+			// map to error condition objects
+			.map((childReference): ParseError => {
+				const { errorCondition } = childReference;
+
+				// `must` always be `true` because of the `filter` call above
+				assertDefined(
+					errorCondition,
+					`Error condition must be present for '${childReference.uri.path}'.`,
+				);
+
+				return errorCondition;
+			});
+		result.push(...childErrorConditions);
+
+		return result;
+	}
+
+	/**
+	 * The top most error of the current reference or any of its
+	 * possible child reference errors.
+	 */
+	public get topError(): IResolveError | undefined {
+		// get all errors, including error of this object
+		const errors = [];
+		if (this.errorCondition) {
+			errors.push(this.errorCondition);
+		}
+		errors.push(...this.allErrors);
+
+		// if no errors, nothing to do
+		if (errors.length === 0) {
+			return undefined;
+		}
+
+
+		// if the first error is the error of the root reference,
+		// then return it as an `error` otherwise use `warning`
+		const [firstError, ...restErrors] = errors;
+		const isRootError = (firstError === this.errorCondition);
+
+		// if a child error - the error is somewhere in the nested references tree,
+		// then use message prefix to highlight that this is not a root error
+		const prefix = (!isRootError)
+			? `${errorMessages.brokenChild}: `
+			: '';
+
+		const moreSuffix = restErrors.length > 0
+			? `\n-\n +${restErrors.length} more error${restErrors.length > 1 ? 's' : ''}`
+			: '';
+
+		const errorMessage = this.getErrorMessage(firstError);
+		return {
+			isRootError,
+			message: `${prefix}${errorMessage}${moreSuffix}`,
+		};
+	}
+
+	/**
+	 * Get message for the provided error condition object.
+	 *
+	 * @param error Error object.
+	 * @returns Error message.
+	 */
+	protected getErrorMessage(error: ParseError): string {
+		// if failed to resolve prompt contents stream, return
+		// the approprivate message and the prompt path
+		if (error instanceof FailedToResolveContentsStream) {
+			return `${errorMessages.streamOpenFailed} '${error.uri.path}'.`;
+		}
+
+		// if a recursion, provide the entire recursion path so users
+		// can use it for the debugging purposes
+		if (error instanceof RecursiveReference) {
+			const { recursivePath } = error;
+
+			const recursivePathString = recursivePath
+				.map((path) => {
+					return basename(URI.file(path));
+				})
+				.join(' -> ');
+
+			return `${errorMessages.recursion}:\n${recursivePathString}`;
+		}
+
+		return error.message;
+	}
+
 	/**
 	 * Check if the current reference points to a given resource.
 	 */
@@ -368,6 +553,7 @@ export abstract class BasePromptParser<T extends IPromptContentsProvider> extend
 		}
 
 		this.disposeReferences();
+		this.stream?.dispose();
 		this._onUpdate.fire();
 
 		super.dispose();
@@ -400,6 +586,23 @@ export class PromptFileReference extends BasePromptParser<FilePromptContentProvi
 		super(provider, seenReferences, initService, configService, logService);
 	}
 
+	/**
+	 * Get the range of the `link` part of the reference.
+	 */
+	public get linkRange(): IRange | undefined {
+		// `#file:` references
+		if (this.token instanceof FileReference) {
+			return this.token.linkRange;
+		}
+
+		// `markdown link` references
+		if (this.token instanceof MarkdownLink) {
+			return this.token.linkRange;
+		}
+
+		return undefined;
+	}
+
 	/**
 	 * Returns a string representation of this object.
 	 */
@@ -410,4 +613,50 @@ export class PromptFileReference extends BasePromptParser<FilePromptContentProvi
 
 		return `${prefix}${this.uri.path}`;
 	}
+
+	/**
+	 * @inheritdoc
+	 */
+	protected override getErrorMessage(error: ParseError): string {
+		// if failed to open a file, return approprivate message and the file path
+		if (error instanceof FileOpenFailed) {
+			return `${errorMessages.fileOpenFailed} '${error.uri.path}'.`;
+		}
+
+		return super.getErrorMessage(error);
+	}
+}
+
+/**
+ * A tiny utility object that helps us to track existance
+ * of at least one parse result from the content provider.
+ */
+class FirstParseResult extends DeferredPromise<void> {
+	/**
+	 * Private attribute to track if we have
+	 * received at least one result.
+	 */
+	private _gotResult = false;
+
+	/**
+	 * Whether we've received at least one result.
+	 */
+	public get gotFirstResult(): boolean {
+		return this._gotResult;
+	}
+
+	/**
+	 * Get underlying promise reference.
+	 */
+	public get promise(): Promise<void> {
+		return this.p;
+	}
+
+	/**
+	 * Complete the underlying promise.
+	 */
+	public override complete() {
+		this._gotResult = true;
+		return super.complete(void 0);
+	}
 }

src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts

@@ -11,8 +11,8 @@ import { IConfigurationService } from '../../../../../../platform/configuration/
 import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js';
 
 /**
- * Class capable of parsing prompt syntax out of a provided text
- * model, including all the nested child file references it may have.
+ * Class capable of parsing prompt syntax out of a provided text model,
+ * including all the nested child file references it may have.
  */
 export class TextModelPromptParser extends BasePromptParser<TextModelContentsProvider> {
 	constructor(

src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.d.ts

@@ -6,6 +6,23 @@
 import { URI } from '../../../../../../base/common/uri.js';
 import { ParseError } from '../../promptFileReferenceErrors.js';
 import { IDisposable } from '../../../../../../base/common/lifecycle.js';
+import { IRange, Range } from '../../../../../../editor/common/core/range.js';
+
+/**
+ * Interface for a resolve error.
+ */
+export interface IResolveError {
+	/**
+	 * Localized error message.
+	 */
+	message: string;
+
+	/**
+	 * Whether this error is for the root reference
+	 * object, or for one of its possible children.
+	 */
+	isRootError: boolean;
+}
 
 /**
  * List of all available prompt reference types.
@@ -20,11 +37,24 @@ export interface IPromptReference extends IDisposable {
 	 * Type of the prompt reference.
 	 */
 	readonly type: PromptReferenceTypes;
+
 	/**
 	 * URI component of the associated with this reference.
 	 */
 	readonly uri: URI;
 
+	/**
+	 * The full range of the prompt reference in the source text,
+	 * including the {@linkcode linkRange} and any additional
+	 * parts the reference may contain (e.g., the `#file:` prefix).
+	 */
+	readonly range: Range;
+
+	/**
+	 * Range of the link part that the reference points to.
+	 */
+	readonly linkRange: IRange | undefined;
+
 	/**
 	 * Flag that indicates if resolving this reference failed.
 	 * The `undefined` means that no attempt to resolve the reference
@@ -35,18 +65,30 @@ export interface IPromptReference extends IDisposable {
 	readonly resolveFailed: boolean | undefined;
 
 	/**
-	 * If failed to resolve the reference this property contains an error
-	 * object that describes the failure reason.
+	 * If failed to resolve the reference this property contains
+	 * an error object that describes the failure reason.
 	 *
 	 * See also {@linkcode resolveFailed}.
 	 */
 	readonly errorCondition: ParseError | undefined;
 
+	/**
+	 * List of all errors that occurred while resolving the current
+	 * reference including all possible errors of nested children.
+	 */
+	readonly allErrors: readonly ParseError[];
+
+	/**
+	 * The top most error of the current reference or any of its
+	 * possible child reference errors.
+	 */
+	readonly topError: IResolveError | undefined;
+
 	/**
 	 * All references that the current reference may have,
 	 * including the all possible nested child references.
 	 */
-	allReferences: readonly IPromptFileReference[];
+	allReferences: readonly IPromptReference[];
 
 	/**
 	 * All *valid* references that the current reference may have,
@@ -56,7 +98,23 @@ export interface IPromptReference extends IDisposable {
 	 * without creating a circular reference loop or having any other
 	 * issues that would make the reference resolve logic to fail.
 	 */
-	allValidReferences: readonly IPromptFileReference[];
+	allValidReferences: readonly IPromptReference[];
+
+	/**
+	 * Returns a promise that resolves when the reference contents
+	 * are completely parsed and all existing tokens are returned.
+	 */
+	settled(): Promise<this>;
+
+	/**
+	 * Returns a promise that resolves when the reference contents,
+	 * and contents for all possible nested child references are
+	 * completely parsed and entire tree of references is built.
+	 *
+	 * The same as {@linkcode settled} but for all prompts in
+	 * the reference tree.
+	 */
+	settledAll(): Promise<this>;
 }
 
 /**

src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/tokens/fileReference.test.ts

@@ -0,0 +1,89 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert from 'assert';
+import { randomInt } from '../../../../../../../../base/common/numbers.js';
+import { Range } from '../../../../../../../../editor/common/core/range.js';
+import { assertDefined } from '../../../../../../../../base/common/types.js';
+import { FileReference } from '../../../../../common/promptSyntax/codecs/tokens/fileReference.js';
+import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../base/test/common/utils.js';
+import { BaseToken } from '../../../../../../../../editor/common/codecs/baseToken.js';
+
+suite('FileReference', () => {
+	ensureNoDisposablesAreLeakedInTestSuite();
+
+	test('`linkRange`', () => {
+		const lineNumber = randomInt(100, 1);
+		const columnStartNumber = randomInt(100, 1);
+		const path = `/temp/test/file-${randomInt(Number.MAX_SAFE_INTEGER)}.txt`;
+		const columnEndNumber = columnStartNumber + path.length;
+
+		const range = new Range(
+			lineNumber,
+			columnStartNumber,
+			lineNumber,
+			columnEndNumber,
+		);
+		const fileReference = new FileReference(range, path);
+		const { linkRange } = fileReference;
+
+		assertDefined(
+			linkRange,
+			'The link range must be defined.',
+		);
+
+		const expectedLinkRange = new Range(
+			lineNumber,
+			columnStartNumber + '#file:'.length,
+			lineNumber,
+			columnStartNumber + path.length,
+		);
+		assert(
+			expectedLinkRange.equalsRange(linkRange),
+			`Expected link range to be ${expectedLinkRange}, got ${linkRange}.`,
+		);
+	});
+
+	test('`path`', () => {
+		const lineNumber = randomInt(100, 1);
+		const columnStartNumber = randomInt(100, 1);
+		const link = `/temp/test/file-${randomInt(Number.MAX_SAFE_INTEGER)}.txt`;
+		const columnEndNumber = columnStartNumber + link.length;
+
+		const range = new Range(
+			lineNumber,
+			columnStartNumber,
+			lineNumber,
+			columnEndNumber,
+		);
+		const fileReference = new FileReference(range, link);
+
+		assert.strictEqual(
+			fileReference.path,
+			link,
+			'Must return the correct link path.',
+		);
+	});
+
+	test('extends `BaseToken`', () => {
+		const lineNumber = randomInt(100, 1);
+		const columnStartNumber = randomInt(100, 1);
+		const link = `/temp/test/file-${randomInt(Number.MAX_SAFE_INTEGER)}.txt`;
+		const columnEndNumber = columnStartNumber + link.length;
+
+		const range = new Range(
+			lineNumber,
+			columnStartNumber,
+			lineNumber,
+			columnEndNumber,
+		);
+		const fileReference = new FileReference(range, link);
+
+		assert(
+			fileReference instanceof BaseToken,
+			'Must extend `BaseToken`.',
+		);
+	});
+});

src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/tokens/markdownLink.test.ts

@@ -0,0 +1,98 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert from 'assert';
+import { randomInt } from '../../../../../../../../base/common/numbers.js';
+import { Range } from '../../../../../../../../editor/common/core/range.js';
+import { assertDefined } from '../../../../../../../../base/common/types.js';
+import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../base/test/common/utils.js';
+import { MarkdownLink } from '../../../../../../../../editor/common/codecs/markdownCodec/tokens/markdownLink.js';
+import { BaseToken } from '../../../../../../../../editor/common/codecs/baseToken.js';
+import { MarkdownToken } from '../../../../../../../../editor/common/codecs/markdownCodec/tokens/markdownToken.js';
+
+suite('FileReference', () => {
+	ensureNoDisposablesAreLeakedInTestSuite();
+
+	test('`linkRange`', () => {
+		const lineNumber = randomInt(100, 1);
+		const columnStartNumber = randomInt(100, 1);
+		const caption = `[link-caption-${randomInt(Number.MAX_SAFE_INTEGER)}]`;
+		const link = `(/temp/test/file-${randomInt(Number.MAX_SAFE_INTEGER)}.md)`;
+
+		const markdownLink = new MarkdownLink(
+			lineNumber,
+			columnStartNumber,
+			caption,
+			link,
+		);
+		const { linkRange } = markdownLink;
+
+		assertDefined(
+			linkRange,
+			'The link range must be defined.',
+		);
+
+		const expectedLinkRange = new Range(
+			lineNumber,
+			// `+1` for the openning `(` character of the link
+			columnStartNumber + caption.length + 1,
+			lineNumber,
+			// `+1` for the openning `(` character of the link, and
+			// `-2` for the enclosing `()` part of the link
+			columnStartNumber + caption.length + 1 + link.length - 2,
+		);
+		assert(
+			expectedLinkRange.equalsRange(linkRange),
+			`Expected link range to be ${expectedLinkRange}, got ${linkRange}.`,
+		);
+	});
+
+	test('`path`', () => {
+		const lineNumber = randomInt(100, 1);
+		const columnStartNumber = randomInt(100, 1);
+		const caption = `[link-caption-${randomInt(Number.MAX_SAFE_INTEGER)}]`;
+		const rawLink = `/temp/test/file-${randomInt(Number.MAX_SAFE_INTEGER)}.md`;
+		const link = `(${rawLink})`;
+
+		const markdownLink = new MarkdownLink(
+			lineNumber,
+			columnStartNumber,
+			caption,
+			link,
+		);
+		const { path } = markdownLink;
+
+		assert.strictEqual(
+			path,
+			rawLink,
+			'Must return the correct link value.',
+		);
+	});
+
+	test('extends `MarkdownToken`', () => {
+		const lineNumber = randomInt(100, 1);
+		const columnStartNumber = randomInt(100, 1);
+		const caption = `[link-caption-${randomInt(Number.MAX_SAFE_INTEGER)}]`;
+		const rawLink = `/temp/test/file-${randomInt(Number.MAX_SAFE_INTEGER)}.md`;
+		const link = `(${rawLink})`;
+
+		const markdownLink = new MarkdownLink(
+			lineNumber,
+			columnStartNumber,
+			caption,
+			link,
+		);
+
+		assert(
+			markdownLink instanceof MarkdownToken,
+			'Must extend `MarkdownToken`.',
+		);
+
+		assert(
+			markdownLink instanceof BaseToken,
+			'Must extend `BaseToken`.',
+		);
+	});
+});

src/vs/workbench/contrib/chat/test/common/promptSyntax/promptFileReference.test.ts

@@ -19,7 +19,7 @@ import { ILogService, NullLogService } from '../../../../../../platform/log/comm
 import { TErrorCondition } from '../../../common/promptSyntax/parsers/basePromptParser.js';
 import { FileReference } from '../../../common/promptSyntax/codecs/tokens/fileReference.js';
 import { FilePromptParser } from '../../../common/promptSyntax/parsers/filePromptParser.js';
-import { wait, waitRandom, randomBoolean } from '../../../../../../base/test/common/testUtils.js';
+import { waitRandom, randomBoolean } from '../../../../../../base/test/common/testUtils.js';
 import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js';
 import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js';
 import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js';
@@ -118,9 +118,8 @@ class TestPromptFileReference extends Disposable {
 			),
 		).start();
 
-		// nested child references are resolved asynchronously in
-		// the background and the process can take some time to complete
-		await wait(50);
+		// wait until entire prompts tree is resolved
+		await rootReference.settledAll();
 
 		// resolve the root file reference including all nested references
 		const resolvedReferences: readonly (IPromptFileReference | undefined)[] = rootReference.allReferences;