diff --git a/src/models/assistant-model.ts b/src/models/assistant-model.ts index 0aec2d8..92753ae 100644 --- a/src/models/assistant-model.ts +++ b/src/models/assistant-model.ts @@ -26,6 +26,15 @@ const OpenAIType = types.custom({ }, }); +interface IGraphAttrData { + legend?: Record; + rightSplit?: Record; + topSplit?: Record; + xAxis?: Record; + yAxis?: Record; + y2Axis?: Record; +} + /** * AssistantModel encapsulates the AI assistant and its interactions with the user. * It includes properties and methods for configuring the assistant, handling chat interactions, and maintaining the assistant's @@ -62,9 +71,10 @@ export const AssistantModel = types transcriptStore: ChatTranscriptModel }) .volatile(() => ({ - updateSonificationStoreAfterRun: false, + dataContextForGraph: null as IGraphAttrData | null, + dataUri: "", uploadFileAfterRun: false, - dataUri: "" + updateSonificationStoreAfterRun: false, })) .actions((self) => ({ addDavaiMsg(msg: string) { @@ -133,7 +143,6 @@ export const AssistantModel = types } }); - const sendDataCtxChangeInfo = flow(function* (msg: string) { try { if (self.isLoadingResponse || self.isCancelling || self.isResetting) { @@ -240,6 +249,7 @@ export const AssistantModel = types yield sendFileMessage(fileId); self.uploadFileAfterRun = false; self.dataUri = ""; + self.dataContextForGraph = null; startRun(); } else { const messages = yield self.apiConnection.beta.threads.messages.list(self.thread.id); @@ -283,6 +293,56 @@ export const AssistantModel = types } }); + const getAttributeData = flow(function* (graphID: string, attrID: string | null) { + if (!attrID) return { attributeData: null }; + + const response = yield Promise.resolve(codapInterface.sendRequest({ + action: "get", + resource: `component[${graphID}].attribute[${attrID}]` + })); + + return response?.values + ? { + id: response.values.id, + name: response.values.name, + values: response.values._categoryMap.__order + } + : null; + }); + + const getGraphAttrData = flow(function* (graphID) { + try { + const graph = yield getGraphByID(graphID); + if (graph) { + const legendAttrData = yield getAttributeData(graphID, graph.legendAttributeID); + const rightAttrData = yield getAttributeData(graphID, graph.rightSplitAttributeID); + const topAttrData = yield getAttributeData(graphID, graph.topSplitAttributeID); + const xAttrData = yield getAttributeData(graphID, graph.xAttributeID); + const yAttrData = yield getAttributeData(graphID, graph.yAttributeID); + const y2AttrData = yield getAttributeData(graphID, graph.y2AttributeID); + + const graphAttrData: IGraphAttrData = { + legend: { attributeData: legendAttrData }, + rightSplit: { attributeData: rightAttrData }, + topSplit: { attributeData: topAttrData }, + xAxis: { attributeData: xAttrData }, + yAxis: { attributeData: yAttrData }, + y2Axis: { attributeData: y2AttrData } + }; + + self.addDbgMsg("Data context for graph", formatJsonMessage(graphAttrData)); + return graphAttrData; + } else { + self.addDbgMsg("No graph found with ID", graphID); + return null; + } + } catch (err) { + console.error("Failed to get graph attribute data:", err); + self.addDbgMsg("Failed to get graph attribute data", formatJsonMessage(err)); + return null; + } + }); + const sendFileMessage = flow(function* (fileId) { try { const res = yield self.apiConnection.beta.threads.messages.create(self.thread.id, { @@ -297,6 +357,10 @@ export const AssistantModel = types image_file: { file_id: fileId } + }, + { + type: "text", + text: `The following JSON data describes key aspects of the graph in the image. Use this context to improve your interpretation and explanation of the graph. ${JSON.stringify(self.dataContextForGraph)}` } ] }); @@ -336,6 +400,15 @@ export const AssistantModel = types if (isImageSnapshotRequest) { self.uploadFileAfterRun = true; self.dataUri = res.values.exportDataUri; + const graphIdMatch = resource.match(/\[(\d+)\]/); + const graphID = graphIdMatch?.[1]; + if (graphID) { + // Send data for the attributes on the graph for additional context + self.dataContextForGraph = yield getGraphAttrData(graphID); + } else { + self.addDbgMsg("Could not extract graphID from resource string", resource); + self.dataContextForGraph = null; + } } // remove any exportDataUri value that exists since it can be large and we don't need to send it to the assistant res = isImageSnapshotRequest