louis_lifu
/
vscode
mirror of https://github.com/microsoft/vscode.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support drag n drop of notebook outputs into chat (#246354) 

* wip

* resolve import shenanigans

* cleanup

* extract making attachment to a contributed util
This commit is contained in:
Michael Lively 2025-04-11 16:49:51 -07:00 committed by GitHub
parent 8e6295b2d5
commit da19de421b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 144 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/vs/platform/dnd/browser/dnd.ts
View File

@ -33,6 +33,7 @@ export const CodeDataTransfers = {
FILES: 'CodeFiles',
SYMBOLS: 'application/vnd.code.symbols',
MARKERS: 'application/vnd.code.diagnostics',
NOTEBOOK_CELL_OUTPUT: 'notebook-cell-output',
};
export interface IDraggedResourceEditorInput extends IBaseTextResourceEditorInput {
@ -416,6 +417,10 @@ export interface DocumentSymbolTransferData {
kind: number;
}
export interface NotebookCellOutputTransferData {
outputId: string;
}
function setDataAsJSON(e: DragEvent, kind: string, data: unknown) {
e.dataTransfer?.setData(kind, JSON.stringify(data));
}
@ -451,6 +456,10 @@ export function fillInMarkersDragData(markerData: MarkerTransferData[], e: DragE
setDataAsJSON(e, CodeDataTransfers.MARKERS, markerData);
}
export function extractNotebookCellOutputDropData(e: DragEvent): NotebookCellOutputTransferData | undefined {
return getDataAsJSON(e, CodeDataTransfers.NOTEBOOK_CELL_OUTPUT, undefined);
}
/**
* A helper to get access to Electrons `webUtils.getPathForFile` function
* in a safe way without crashing the application when running in the web.

32
src/vs/workbench/contrib/chat/browser/chatAttachmentResolve.ts
View File

@ -13,7 +13,7 @@ import { SymbolKinds } from '../../../../editor/common/languages.js';
import { ITextModelService } from '../../../../editor/common/services/resolverService.js';
import { localize } from '../../../../nls.js';
import { IDialogService } from '../../../../platform/dialogs/common/dialogs.js';
import { IDraggedResourceEditorInput, MarkerTransferData, DocumentSymbolTransferData } from '../../../../platform/dnd/browser/dnd.js';
import { IDraggedResourceEditorInput, MarkerTransferData, DocumentSymbolTransferData, NotebookCellOutputTransferData } from '../../../../platform/dnd/browser/dnd.js';
import { IFileService } from '../../../../platform/files/common/files.js';
import { MarkerSeverity } from '../../../../platform/markers/common/markers.js';
import { isUntitledResourceEditorInput } from '../../../common/editor.js';
@ -21,6 +21,9 @@ import { EditorInput } from '../../../common/editor/editorInput.js';
import { IEditorService } from '../../../services/editor/common/editorService.js';
import { IExtensionService, isProposedApiEnabled } from '../../../services/extensions/common/extensions.js';
import { UntitledTextEditorInput } from '../../../services/untitled/common/untitledTextEditorInput.js';
import { createNotebookOutputVariableEntry, NOTEBOOK_CELL_OUTPUT_MIME_TYPE_LIST_FOR_CHAT_CONST } from '../../notebook/browser/contrib/chat/notebookChatUtils.js';
import { getOutputViewModelFromId } from '../../notebook/browser/controller/cellOutputActions.js';
import { getNotebookEditorFromEditorPane } from '../../notebook/browser/notebookBrowser.js';
import { IChatRequestVariableEntry, IDiagnosticVariableEntry, IDiagnosticVariableEntryFilterData, ISymbolVariableEntry, OmittedState } from '../common/chatModel.js';
import { imageToHash } from './chatPasteProviders.js';
import { resizeImage } from './imageUtils.js';
@ -232,3 +235,30 @@ function symbolId(resource: URI, range?: IRange): string {
}
return resource.fsPath + rangePart;
}
// --- NOTEBOOKS ---
export function resolveNotebookOutputAttachContext(data: NotebookCellOutputTransferData, editorService: IEditorService): IChatRequestVariableEntry[] {
const notebookEditor = getNotebookEditorFromEditorPane(editorService.activeEditorPane);
if (!notebookEditor) {
return [];
}
const outputViewModel = getOutputViewModelFromId(data.outputId, notebookEditor);
if (!outputViewModel) {
return [];
}
const mimeType = outputViewModel.pickedMimeType?.mimeType;
if (mimeType && NOTEBOOK_CELL_OUTPUT_MIME_TYPE_LIST_FOR_CHAT_CONST.includes(mimeType)) {
const entry = createNotebookOutputVariableEntry(outputViewModel, mimeType, notebookEditor);
if (!entry) {
return [];
}
return [entry];
}
return [];
}

19
src/vs/workbench/contrib/chat/browser/chatDragAndDrop.ts
View File

@ -16,7 +16,7 @@ import { URI } from '../../../../base/common/uri.js';
import { ITextModelService } from '../../../../editor/common/services/resolverService.js';
import { localize } from '../../../../nls.js';
import { IDialogService } from '../../../../platform/dialogs/common/dialogs.js';
import { CodeDataTransfers, containsDragType, extractEditorsDropData, extractMarkerDropData, extractSymbolDropData } from '../../../../platform/dnd/browser/dnd.js';
import { CodeDataTransfers, containsDragType, extractEditorsDropData, extractMarkerDropData, extractNotebookCellOutputDropData, extractSymbolDropData } from '../../../../platform/dnd/browser/dnd.js';
import { IFileService } from '../../../../platform/files/common/files.js';
import { ILogService } from '../../../../platform/log/common/log.js';
import { IThemeService, Themable } from '../../../../platform/theme/common/themeService.js';
@ -25,7 +25,7 @@ import { IEditorService } from '../../../services/editor/common/editorService.js
import { IExtensionService, isProposedApiEnabled } from '../../../services/extensions/common/extensions.js';
import { IChatRequestVariableEntry } from '../common/chatModel.js';
import { IChatWidgetService } from './chat.js';
import { ImageTransferData, resolveEditorAttachContext, resolveImageAttachContext, resolveMarkerAttachContext, resolveSymbolsAttachContext } from './chatAttachmentResolve.js';
import { ImageTransferData, resolveEditorAttachContext, resolveImageAttachContext, resolveMarkerAttachContext, resolveNotebookOutputAttachContext, resolveSymbolsAttachContext } from './chatAttachmentResolve.js';
import { ChatAttachmentModel } from './chatAttachmentModel.js';
import { IChatInputStyles } from './chatInputPart.js';
import { convertStringToUInt8Array } from './imageUtils.js';
@ -38,6 +38,7 @@ enum ChatDragAndDropType {
SYMBOL,
HTML,
MARKER,
NOTEBOOK_CELL_OUTPUT
}
const IMAGE_DATA_REGEX = /^data:image\/[a-z]+;base64,/;
@ -168,8 +169,10 @@ export class ChatDragAndDrop extends Themable {
}
private guessDropType(e: DragEvent): ChatDragAndDropType | undefined {
// This is an esstimation based on the datatransfer types/items
if (containsImageDragType(e)) {
// This is an estimation based on the datatransfer types/items
if (containsDragType(e, CodeDataTransfers.NOTEBOOK_CELL_OUTPUT)) {
return ChatDragAndDropType.NOTEBOOK_CELL_OUTPUT;
} else if (containsImageDragType(e)) {
return this.extensionService.extensions.some(ext => isProposedApiEnabled(ext, 'chatReferenceBinaryData')) ? ChatDragAndDropType.IMAGE : undefined;
} else if (containsDragType(e, 'text/html')) {
return ChatDragAndDropType.HTML;
@ -203,6 +206,7 @@ export class ChatDragAndDrop extends Themable {
case ChatDragAndDropType.SYMBOL: return localize('symbol', 'Symbol');
case ChatDragAndDropType.MARKER: return localize('problem', 'Problem');
case ChatDragAndDropType.HTML: return localize('url', 'URL');
case ChatDragAndDropType.NOTEBOOK_CELL_OUTPUT: return localize('notebookOutput', 'Output');
}
}
@ -211,6 +215,13 @@ export class ChatDragAndDrop extends Themable {
return [];
}
if (containsDragType(e, CodeDataTransfers.NOTEBOOK_CELL_OUTPUT)) {
const notebookOutputData = extractNotebookCellOutputDropData(e);
if (notebookOutputData) {
return resolveNotebookOutputAttachContext(notebookOutputData, this.editorService);
}
}
const markerData = extractMarkerDropData(e);
if (markerData) {
return resolveMarkerAttachContext(markerData);

67
src/vs/workbench/contrib/notebook/browser/contrib/chat/notebookChatUtils.ts Normal file
View File

@ -0,0 +1,67 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { normalizeDriveLetter } from '../../../../../../base/common/labels.js';
import { basenameOrAuthority } from '../../../../../../base/common/resources.js';
import { ThemeIcon } from '../../../../../../base/common/themables.js';
import { localize } from '../../../../../../nls.js';
import { INotebookOutputVariableEntry } from '../../../../chat/common/chatModel.js';
import { CellUri } from '../../../common/notebookCommon.js';
import { ICellOutputViewModel, INotebookEditor } from '../../notebookBrowser.js';
export const NOTEBOOK_CELL_OUTPUT_MIME_TYPE_LIST_FOR_CHAT_CONST = [
'text/plain',
'text/html',
'application/vnd.code.notebook.error',
'application/vnd.code.notebook.stdout',
'application/x.notebook.stdout',
'application/x.notebook.stream',
'application/vnd.code.notebook.stderr',
'application/x.notebook.stderr',
'image/png',
'image/jpeg',
'image/svg',
];
export function createNotebookOutputVariableEntry(outputViewModel: ICellOutputViewModel, mimeType: string, notebookEditor: INotebookEditor): INotebookOutputVariableEntry | undefined {
// get the cell index
const cellFromViewModelHandle = outputViewModel.cellViewModel.handle;
const notebookModel = notebookEditor.textModel;
const cell = notebookEditor.getCellByHandle(cellFromViewModelHandle);
if (!cell || cell.outputsViewModels.length === 0 || !notebookModel) {
return;
}
// uri of the cell
const notebookUri = notebookModel.uri;
const cellUri = cell.uri;
const cellIndex = notebookModel.cells.indexOf(cell.model);
// get the output index
const outputId = outputViewModel?.model.outputId;
let outputIndex: number = 0;
if (outputId !== undefined) {
// find the output index
outputIndex = cell.outputsViewModels.findIndex(output => {
return output.model.outputId === outputId;
});
}
// construct the URI using the cell uri and output index
const outputCellUri = CellUri.generateCellOutputUriWithIndex(notebookUri, cellUri, outputIndex);
const fileName = normalizeDriveLetter(basenameOrAuthority(notebookUri));
const l: INotebookOutputVariableEntry = {
value: outputCellUri,
id: outputCellUri.toString(),
name: localize('notebookOutputCellLabel', "{0} • Cell {1} • Output {2}", fileName, `${cellIndex + 1}`, `${outputIndex + 1}`),
icon: mimeType === 'application/vnd.code.notebook.error' ? ThemeIcon.fromId('error') : undefined,
kind: 'notebookOutput',
outputIndex,
mimeType
};
return l;
}

56
src/vs/workbench/contrib/notebook/browser/controller/chat/notebook.chat.contribution.ts
View File

@ -26,34 +26,19 @@ import { computeCompletionRanges } from '../../../../chat/browser/contrib/chatIn
import { IChatAgentService } from '../../../../chat/common/chatAgents.js';
import { ChatAgentLocation } from '../../../../chat/common/constants.js';
import { ChatContextKeys } from '../../../../chat/common/chatContextKeys.js';
import { INotebookOutputVariableEntry } from '../../../../chat/common/chatModel.js';
import { chatVariableLeader } from '../../../../chat/common/chatParserTypes.js';
import { NOTEBOOK_CELL_HAS_OUTPUTS, NOTEBOOK_CELL_OUTPUT_MIME_TYPE_LIST_FOR_CHAT, NOTEBOOK_CELL_OUTPUT_MIMETYPE } from '../../../common/notebookContextKeys.js';
import { INotebookKernelService } from '../../../common/notebookKernelService.js';
import { getNotebookEditorFromEditorPane, ICellOutputViewModel, INotebookEditor, ICellViewModel } from '../../notebookBrowser.js';
import { getNotebookEditorFromEditorPane, ICellOutputViewModel, INotebookEditor } from '../../notebookBrowser.js';
import * as icons from '../../notebookIcons.js';
import { getOutputViewModelFromId } from '../cellOutputActions.js';
import { INotebookOutputActionContext, NOTEBOOK_ACTIONS_CATEGORY } from '../coreActions.js';
import { CellUri } from '../../../common/notebookCommon.js';
import './cellChatActions.js';
import { CTX_NOTEBOOK_CHAT_HAS_AGENT } from './notebookChatContext.js';
import { IViewsService } from '../../../../../services/views/common/viewsService.js';
import { ThemeIcon } from '../../../../../../base/common/themables.js';
import { normalizeDriveLetter } from '../../../../../../base/common/labels.js';
import { basenameOrAuthority } from '../../../../../../base/common/resources.js';
import { createNotebookOutputVariableEntry, NOTEBOOK_CELL_OUTPUT_MIME_TYPE_LIST_FOR_CHAT_CONST } from '../../contrib/chat/notebookChatUtils.js';
const NotebookKernelVariableKey = 'kernelVariable';
const NOTEBOOK_CELL_OUTPUT_MIME_TYPE_LIST_FOR_CHAT_CONST = ['text/plain', 'text/html',
'application/vnd.code.notebook.error',
'application/vnd.code.notebook.stdout',
'application/x.notebook.stdout',
'application/x.notebook.stream',
'application/vnd.code.notebook.stderr',
'application/x.notebook.stderr',
'image/png',
'image/jpeg',
'image/svg',
];
class NotebookChatContribution extends Disposable implements IWorkbenchContribution {
static readonly ID = 'workbench.contrib.notebookChatContribution';
@ -327,43 +312,12 @@ registerAction2(class CopyCellOutputAction extends Action2 {
}
if (mimeType && NOTEBOOK_CELL_OUTPUT_MIME_TYPE_LIST_FOR_CHAT_CONST.includes(mimeType)) {
// get the cell index
const cellFromViewModelHandle = outputViewModel.cellViewModel.handle;
const notebookModel = notebookEditor.textModel;
const cell: ICellViewModel | undefined = notebookEditor.getCellByHandle(cellFromViewModelHandle);
if (!cell || cell.outputsViewModels.length === 0 || !notebookModel) {
const entry = createNotebookOutputVariableEntry(outputViewModel, mimeType, notebookEditor);
if (!entry) {
return;
}
// uri of the cell
const notebookUri = notebookModel.uri;
const cellUri = cell.uri;
const cellIndex = notebookModel.cells.indexOf(cell.model);
// get the output index
const outputId = outputViewModel?.model.outputId;
let outputIndex: number = 0;
if (outputId !== undefined) {
// find the output index
outputIndex = cell.outputsViewModels.findIndex(output => {
return output.model.outputId === outputId;
});
}
// construct the URI using the cell uri and output index
const outputCellUri = CellUri.generateCellOutputUriWithIndex(notebookUri, cellUri, outputIndex);
const fileName = normalizeDriveLetter(basenameOrAuthority(notebookUri));
const l: INotebookOutputVariableEntry = {
value: outputCellUri,
id: outputCellUri.toString(),
name: localize('notebookOutputCellLabel', "{0} • Cell {1} • Output {2}", fileName, `${cellIndex + 1}`, `${outputIndex + 1}`),
icon: mimeType === 'application/vnd.code.notebook.error' ? ThemeIcon.fromId('error') : undefined,
kind: 'notebookOutput',
outputIndex,
mimeType
};
widget.attachmentModel.addContext(l);
widget.attachmentModel.addContext(entry);
(await showChatView(viewService))?.focusInput();
}
}

17
src/vs/workbench/contrib/notebook/browser/view/renderers/webviewPreloads.ts
View File

@ -8,6 +8,7 @@ import type { IDisposable } from '../../../../../../base/common/lifecycle.js';
import type * as webviewMessages from './webviewMessages.js';
import type { NotebookCellMetadata } from '../../../common/notebookCommon.js';
import type * as rendererApi from 'vscode-notebook-renderer';
import type { NotebookCellOutputTransferData } from '../../../../../../platform/dnd/browser/dnd.js';
// !! IMPORTANT !! ----------------------------------------------------------------------------------
// import { RenderOutputType } from 'vs/workbench/contrib/notebook/browser/notebookBrowser';
@ -2908,12 +2909,28 @@ async function webviewPreloads(ctx: PreloadContext) {
this.element.style.left = left + 'px';
this.element.style.padding = `${ctx.style.outputNodePadding}px ${ctx.style.outputNodePadding}px ${ctx.style.outputNodePadding}px ${ctx.style.outputNodeLeftPadding}`;
// Make output draggable
this.element.draggable = true;
this.element.addEventListener('mouseenter', () => {
postNotebookMessage<webviewMessages.IMouseEnterMessage>('mouseenter', { id: outputId });
});
this.element.addEventListener('mouseleave', () => {
postNotebookMessage<webviewMessages.IMouseLeaveMessage>('mouseleave', { id: outputId });
});
// Add drag handler
this.element.addEventListener('dragstart', (e: DragEvent) => {
if (!e.dataTransfer) {
return;
}
const outputData: NotebookCellOutputTransferData = {
outputId: this.outputId,
};
e.dataTransfer.setData('notebook-cell-output', JSON.stringify(outputData));
});
}
public dispose() {