Skip to content

Commit f504c56

Browse files
committed
feat: link data to graph image for description (DAVAI-38)
[DAVAI-38](https://concord-consortium.atlassian.net/browse/DAVAI-38)
1 parent f6a5e95 commit f504c56

File tree

2 files changed

+83
-4
lines changed

2 files changed

+83
-4
lines changed

src/app-config.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"accessibility": {
33
"keyboardShortcut": "ctrl+?"
44
},
5-
"assistantId": "asst_xmAX5oxByssXrkBymMbcsVEm",
5+
"assistantId": "asst_2sNDbextdsj0E3E0YugE08QH",
66
"dimensions": {
77
"height": 680,
88
"width": 380

src/models/assistant-model.ts

Lines changed: 82 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,14 @@ const OpenAIType = types.custom({
2626
},
2727
});
2828

29+
interface IGraphAttrData {
30+
legend?: Record<string, any>;
31+
rightSplit?: Record<string, any>;
32+
topSplit?: Record<string, any>;
33+
xAxis?: Record<string, any>;
34+
yAxis?: Record<string, any>;
35+
}
36+
2937
/**
3038
* AssistantModel encapsulates the AI assistant and its interactions with the user.
3139
* It includes properties and methods for configuring the assistant, handling chat interactions, and maintaining the assistant's
@@ -62,9 +70,10 @@ export const AssistantModel = types
6270
transcriptStore: ChatTranscriptModel
6371
})
6472
.volatile(() => ({
65-
updateSonificationStoreAfterRun: false,
73+
dataUri: "",
74+
dataContextForGraph: null as IGraphAttrData | null,
6675
uploadFileAfterRun: false,
67-
dataUri: ""
76+
updateSonificationStoreAfterRun: false,
6877
}))
6978
.actions((self) => ({
7079
addDavaiMsg(msg: string) {
@@ -133,7 +142,6 @@ export const AssistantModel = types
133142
}
134143
});
135144

136-
137145
const sendDataCtxChangeInfo = flow(function* (msg: string) {
138146
try {
139147
if (self.isLoadingResponse || self.isCancelling || self.isResetting) {
@@ -240,6 +248,7 @@ export const AssistantModel = types
240248
yield sendFileMessage(fileId);
241249
self.uploadFileAfterRun = false;
242250
self.dataUri = "";
251+
self.dataContextForGraph = null;
243252
startRun();
244253
} else {
245254
const messages = yield self.apiConnection.beta.threads.messages.list(self.thread.id);
@@ -283,6 +292,67 @@ export const AssistantModel = types
283292
}
284293
});
285294

295+
const getAttributeData = flow(function* (graphID: string, attrID: string | null) {
296+
if (!attrID) return { attributeData: null };
297+
298+
const response = yield Promise.resolve(codapInterface.sendRequest({
299+
action: "get",
300+
resource: `component[${graphID}].attribute[${attrID}]`
301+
}));
302+
303+
const attributeData = response?.values
304+
? {
305+
id: response.values.id,
306+
name: response.values.name,
307+
values: response.values._categoryMap.__order
308+
}
309+
: null;
310+
311+
return { attributeData };
312+
});
313+
314+
const getGraphAttrData = flow(function* (graphID) {
315+
try {
316+
const graph = yield getGraphByID(graphID);
317+
if (graph) {
318+
const legendAttrData = yield getAttributeData(graphID, graph.legendAttributeID);
319+
const rightAttrData = yield getAttributeData(graphID, graph.rightSplitAttributeID);
320+
const topAttrData = yield getAttributeData(graphID, graph.topSplitAttributeID);
321+
const xAttrData = yield getAttributeData(graphID, graph.xAttributeID);
322+
const yAttrData = yield getAttributeData(graphID, graph.yAttributeID);
323+
const y2AttrData = yield getAttributeData(graphID, graph.yAttributeIDs ? graph.yAttributeIDs[1] : null);
324+
325+
// Combine y-axis data if we have a second y-axis
326+
const combinedYAxisData = y2AttrData.attributeData
327+
? { attributeData: [yAttrData.attributeData, y2AttrData.attributeData] }
328+
: yAttrData;
329+
330+
const graphAttrData = {
331+
legend: legendAttrData,
332+
rightSplit: rightAttrData,
333+
topSplit: topAttrData,
334+
xAxis: xAttrData,
335+
yAxis: combinedYAxisData
336+
};
337+
338+
if (graphAttrData) {
339+
self.addDbgMsg("Data context for graph", formatJsonMessage(graphAttrData));
340+
return graphAttrData;
341+
} else {
342+
self.addDbgMsg("No data context found for graph", formatJsonMessage(graph));
343+
return null;
344+
}
345+
} else {
346+
self.addDbgMsg("No graph found with ID", graphID);
347+
return null;
348+
}
349+
} catch (err) {
350+
console.error("Failed to get graph attribute data:", err);
351+
self.addDbgMsg("Failed to get graph attribute data", formatJsonMessage(err));
352+
return null;
353+
}
354+
});
355+
286356
const sendFileMessage = flow(function* (fileId) {
287357
try {
288358
const res = yield self.apiConnection.beta.threads.messages.create(self.thread.id, {
@@ -297,6 +367,10 @@ export const AssistantModel = types
297367
image_file: {
298368
file_id: fileId
299369
}
370+
},
371+
{
372+
type: "text",
373+
text: `The following JSON data describes key aspects of the graph in the image. Use this context to improve your interpretation and explanation of the graph. ${JSON.stringify(self.dataContextForGraph)}`
300374
}
301375
]
302376
});
@@ -336,6 +410,11 @@ export const AssistantModel = types
336410
if (isImageSnapshotRequest) {
337411
self.uploadFileAfterRun = true;
338412
self.dataUri = res.values.exportDataUri;
413+
const graphID = resource.match(/\[(\d+)\]/)?.[1];
414+
// We'll also send data for the attributes on the graph for additional context
415+
self.dataContextForGraph = yield getGraphAttrData(graphID);
416+
// TODO: Remove this console log when done with testing
417+
console.log("Attribute data for graph", self.dataContextForGraph);
339418
}
340419
// remove any exportDataUri value that exists since it can be large and we don't need to send it to the assistant
341420
res = isImageSnapshotRequest

0 commit comments

Comments
 (0)