Skip to content

Commit 80160ee

Browse files
committed
feat: link data to graph image for description (DAVAI-38)
[DAVAI-38](https://concord-consortium.atlassian.net/browse/DAVAI-38)
1 parent f027e1a commit 80160ee

File tree

2 files changed

+94
-12
lines changed

2 files changed

+94
-12
lines changed

src/app-config.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"accessibility": {
33
"keyboardShortcut": "ctrl+?"
44
},
5-
"assistantId": "asst_xmAX5oxByssXrkBymMbcsVEm",
5+
"assistantId": "asst_2sNDbextdsj0E3E0YugE08QH",
66
"dimensions": {
77
"height": 680,
88
"width": 380

src/models/assistant-model.ts

Lines changed: 93 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,25 +26,34 @@ const OpenAIType = types.custom({
2626
},
2727
});
2828

29+
interface IGraphAttrData {
30+
legend?: Record<string, any>;
31+
rightSplit?: Record<string, any>;
32+
topSplit?: Record<string, any>;
33+
xAxis?: Record<string, any>;
34+
yAxis?: Record<string, any>;
35+
}
36+
2937
/**
3038
* AssistantModel encapsulates the AI assistant and its interactions with the user.
3139
* It includes properties and methods for configuring the assistant, handling chat interactions, and maintaining the assistant's
3240
* thread and transcript.
3341
*
42+
* @property {Object} apiConnection - The API connection object for interacting with the assistant
3443
* @property {Object|null} assistant - The assistant object, or `null` if not initialized.
3544
* @property {string} assistantId - The unique ID of the assistant being used, or `null` if not initialized.
36-
* @property {Object} apiConnection - The API connection object for interacting with the assistant
37-
* @property {Object|null} thread - The assistant's thread used for the current chat, or `null` if no thread is active.
38-
* @property {ChatTranscriptModel} transcriptStore - The assistant's chat transcript store for recording and managing chat messages.
39-
* @property {boolean} isLoadingResponse - Flag indicating whether the assistant is currently processing a response.
45+
* @property {Object} assistantList - A map of available assistants, where the key is the assistant ID and the value is the assistant name.
4046
* @property {string[]} codapNotificationQueue - Queue of messages to be sent to the assistant. Used if CODAP generates notifications while assistant is processing a response.
41-
* @property {string[]} messageQueue - Queue of messages to be sent to the assistant. Used if user sends messages while assistant is processing a response.
42-
* @property {boolean} showLoadingIndicator - Flag indicating whether to show a loading indicator to the user; this is decoupled from the assistant's internal loading state to allow for more control over UI elements.
47+
* @property {string} dataUri - The data URI of the file to be uploaded.
48+
* @property {string} dataContextForGraph - The data context for the graph being processed by the assistant.
4349
* @property {boolean} isCancelling - Flag indicating whether the assistant is currently cancelling a request.
50+
* @property {boolean} isLoadingResponse - Flag indicating whether the assistant is currently processing a response.
4451
* @property {boolean} isResetting - Flag indicating whether the assistant is currently resetting the chat.
52+
* @property {string[]} messageQueue - Queue of messages to be sent to the assistant. Used if user sends messages while assistant is processing a response.
53+
* @property {boolean} showLoadingIndicator - Flag indicating whether to show a loading indicator to the user; this is decoupled from the assistant's internal loading state to allow for more control over UI elements.
54+
* @property {Object|null} thread - The assistant's thread used for the current chat, or `null` if no thread is active.
55+
* @property {ChatTranscriptModel} transcriptStore - The assistant's chat transcript store for recording and managing chat messages.
4556
* @property {boolean} uploadFileAfterRun - Flag indicating whether to upload a file after the assistant completes a run.
46-
* @property {string} dataUri - The data URI of the file to be uploaded.
47-
* @property {string} graphToSonify - The name of the graph to sonify, if applicable.
4857
*/
4958
export const AssistantModel = types
5059
.model("AssistantModel", {
@@ -60,9 +69,12 @@ export const AssistantModel = types
6069
showLoadingIndicator: types.optional(types.boolean, false),
6170
thread: types.maybe(types.frozen()),
6271
transcriptStore: ChatTranscriptModel,
63-
uploadFileAfterRun: false,
64-
dataUri: ""
72+
uploadFileAfterRun: false
6573
})
74+
.volatile(() => ({
75+
dataUri: "",
76+
dataContextForGraph: null as IGraphAttrData | null
77+
}))
6678
.actions((self) => ({
6779
addDavaiMsg(msg: string) {
6880
self.transcriptStore.addMessage(DAVAI_SPEAKER, { content: msg });
@@ -130,7 +142,6 @@ export const AssistantModel = types
130142
}
131143
});
132144

133-
134145
const sendDataCtxChangeInfo = flow(function* (msg: string) {
135146
try {
136147
if (self.isLoadingResponse || self.isCancelling || self.isResetting) {
@@ -237,6 +248,7 @@ export const AssistantModel = types
237248
yield sendFileMessage(fileId);
238249
self.uploadFileAfterRun = false;
239250
self.dataUri = "";
251+
self.dataContextForGraph = null;
240252
startRun();
241253
} else {
242254
const messages = yield self.apiConnection.beta.threads.messages.list(self.thread.id);
@@ -274,6 +286,67 @@ export const AssistantModel = types
274286
}
275287
});
276288

289+
const getAttributeData = flow(function* (graphID: string, attrID: string | null) {
290+
if (!attrID) return { attributeData: null };
291+
292+
const response = yield Promise.resolve(codapInterface.sendRequest({
293+
action: "get",
294+
resource: `component[${graphID}].attribute[${attrID}]`
295+
}));
296+
297+
const attributeData = response?.values
298+
? {
299+
id: response.values.id,
300+
name: response.values.name,
301+
values: response.values._categoryMap.__order
302+
}
303+
: null;
304+
305+
return { attributeData };
306+
});
307+
308+
const getGraphAttrData = flow(function* (graphID) {
309+
try {
310+
const graph = yield getGraphByID(graphID);
311+
if (graph) {
312+
const legendAttrData = yield getAttributeData(graphID, graph.legendAttributeID);
313+
const rightAttrData = yield getAttributeData(graphID, graph.rightSplitAttributeID);
314+
const topAttrData = yield getAttributeData(graphID, graph.topSplitAttributeID);
315+
const xAttrData = yield getAttributeData(graphID, graph.xAttributeID);
316+
const yAttrData = yield getAttributeData(graphID, graph.yAttributeID);
317+
const y2AttrData = yield getAttributeData(graphID, graph.yAttributeIDs ? graph.yAttributeIDs[1] : null);
318+
319+
// Combine y-axis data if we have a second y-axis
320+
const combinedYAxisData = y2AttrData.attributeData
321+
? { attributeData: [yAttrData.attributeData, y2AttrData.attributeData] }
322+
: yAttrData;
323+
324+
const graphAttrData = {
325+
legend: legendAttrData,
326+
rightSplit: rightAttrData,
327+
topSplit: topAttrData,
328+
xAxis: xAttrData,
329+
yAxis: combinedYAxisData
330+
};
331+
332+
if (graphAttrData) {
333+
self.addDbgMsg("Data context for graph", formatJsonMessage(graphAttrData));
334+
return graphAttrData;
335+
} else {
336+
self.addDbgMsg("No data context found for graph", formatJsonMessage(graph));
337+
return null;
338+
}
339+
} else {
340+
self.addDbgMsg("No graph found with ID", graphID);
341+
return null;
342+
}
343+
} catch (err) {
344+
console.error("Failed to get graph attribute data:", err);
345+
self.addDbgMsg("Failed to get graph attribute data", formatJsonMessage(err));
346+
return null;
347+
}
348+
});
349+
277350
const sendFileMessage = flow(function* (fileId) {
278351
try {
279352
const res = yield self.apiConnection.beta.threads.messages.create(self.thread.id, {
@@ -288,6 +361,10 @@ export const AssistantModel = types
288361
image_file: {
289362
file_id: fileId
290363
}
364+
},
365+
{
366+
type: "text",
367+
text: `The following JSON data describes key aspects of the graph in the image. Use this context to improve your interpretation and explanation of the graph. ${JSON.stringify(self.dataContextForGraph)}`
291368
}
292369
]
293370
});
@@ -320,6 +397,11 @@ export const AssistantModel = types
320397
if (isImageSnapshotRequest) {
321398
self.uploadFileAfterRun = true;
322399
self.dataUri = res.values.exportDataUri;
400+
const graphID = resource.match(/\[(\d+)\]/)?.[1];
401+
// We'll also send data for the attributes on the graph for additional context
402+
self.dataContextForGraph = yield getGraphAttrData(graphID);
403+
// TODO: Remove this console log when done with testing
404+
console.log("Attribute data for graph", self.dataContextForGraph);
323405
}
324406
// remove any exportDataUri value that exists since it can be large and we don't need to send it to the assistant
325407
res = isImageSnapshotRequest

0 commit comments

Comments
 (0)