@@ -26,6 +26,14 @@ const OpenAIType = types.custom({
26
26
} ,
27
27
} ) ;
28
28
29
+ interface IGraphAttrData {
30
+ legend ?: Record < string , any > ;
31
+ rightSplit ?: Record < string , any > ;
32
+ topSplit ?: Record < string , any > ;
33
+ xAxis ?: Record < string , any > ;
34
+ yAxis ?: Record < string , any > ;
35
+ }
36
+
29
37
/**
30
38
* AssistantModel encapsulates the AI assistant and its interactions with the user.
31
39
* It includes properties and methods for configuring the assistant, handling chat interactions, and maintaining the assistant's
@@ -62,9 +70,10 @@ export const AssistantModel = types
62
70
transcriptStore : ChatTranscriptModel
63
71
} )
64
72
. volatile ( ( ) => ( {
65
- updateSonificationStoreAfterRun : false ,
73
+ dataUri : "" ,
74
+ dataContextForGraph : null as IGraphAttrData | null ,
66
75
uploadFileAfterRun : false ,
67
- dataUri : ""
76
+ updateSonificationStoreAfterRun : false ,
68
77
} ) )
69
78
. actions ( ( self ) => ( {
70
79
addDavaiMsg ( msg : string ) {
@@ -133,7 +142,6 @@ export const AssistantModel = types
133
142
}
134
143
} ) ;
135
144
136
-
137
145
const sendDataCtxChangeInfo = flow ( function * ( msg : string ) {
138
146
try {
139
147
if ( self . isLoadingResponse || self . isCancelling || self . isResetting ) {
@@ -240,6 +248,7 @@ export const AssistantModel = types
240
248
yield sendFileMessage ( fileId ) ;
241
249
self . uploadFileAfterRun = false ;
242
250
self . dataUri = "" ;
251
+ self . dataContextForGraph = null ;
243
252
startRun ( ) ;
244
253
} else {
245
254
const messages = yield self . apiConnection . beta . threads . messages . list ( self . thread . id ) ;
@@ -283,6 +292,67 @@ export const AssistantModel = types
283
292
}
284
293
} ) ;
285
294
295
+ const getAttributeData = flow ( function * ( graphID : string , attrID : string | null ) {
296
+ if ( ! attrID ) return { attributeData : null } ;
297
+
298
+ const response = yield Promise . resolve ( codapInterface . sendRequest ( {
299
+ action : "get" ,
300
+ resource : `component[${ graphID } ].attribute[${ attrID } ]`
301
+ } ) ) ;
302
+
303
+ const attributeData = response ?. values
304
+ ? {
305
+ id : response . values . id ,
306
+ name : response . values . name ,
307
+ values : response . values . _categoryMap . __order
308
+ }
309
+ : null ;
310
+
311
+ return { attributeData } ;
312
+ } ) ;
313
+
314
+ const getGraphAttrData = flow ( function * ( graphID ) {
315
+ try {
316
+ const graph = yield getGraphByID ( graphID ) ;
317
+ if ( graph ) {
318
+ const legendAttrData = yield getAttributeData ( graphID , graph . legendAttributeID ) ;
319
+ const rightAttrData = yield getAttributeData ( graphID , graph . rightSplitAttributeID ) ;
320
+ const topAttrData = yield getAttributeData ( graphID , graph . topSplitAttributeID ) ;
321
+ const xAttrData = yield getAttributeData ( graphID , graph . xAttributeID ) ;
322
+ const yAttrData = yield getAttributeData ( graphID , graph . yAttributeID ) ;
323
+ const y2AttrData = yield getAttributeData ( graphID , graph . yAttributeIDs ? graph . yAttributeIDs [ 1 ] : null ) ;
324
+
325
+ // Combine y-axis data if we have a second y-axis
326
+ const combinedYAxisData = y2AttrData . attributeData
327
+ ? { attributeData : [ yAttrData . attributeData , y2AttrData . attributeData ] }
328
+ : yAttrData ;
329
+
330
+ const graphAttrData = {
331
+ legend : legendAttrData ,
332
+ rightSplit : rightAttrData ,
333
+ topSplit : topAttrData ,
334
+ xAxis : xAttrData ,
335
+ yAxis : combinedYAxisData
336
+ } ;
337
+
338
+ if ( graphAttrData ) {
339
+ self . addDbgMsg ( "Data context for graph" , formatJsonMessage ( graphAttrData ) ) ;
340
+ return graphAttrData ;
341
+ } else {
342
+ self . addDbgMsg ( "No data context found for graph" , formatJsonMessage ( graph ) ) ;
343
+ return null ;
344
+ }
345
+ } else {
346
+ self . addDbgMsg ( "No graph found with ID" , graphID ) ;
347
+ return null ;
348
+ }
349
+ } catch ( err ) {
350
+ console . error ( "Failed to get graph attribute data:" , err ) ;
351
+ self . addDbgMsg ( "Failed to get graph attribute data" , formatJsonMessage ( err ) ) ;
352
+ return null ;
353
+ }
354
+ } ) ;
355
+
286
356
const sendFileMessage = flow ( function * ( fileId ) {
287
357
try {
288
358
const res = yield self . apiConnection . beta . threads . messages . create ( self . thread . id , {
@@ -297,6 +367,10 @@ export const AssistantModel = types
297
367
image_file : {
298
368
file_id : fileId
299
369
}
370
+ } ,
371
+ {
372
+ type : "text" ,
373
+ text : `The following JSON data describes key aspects of the graph in the image. Use this context to improve your interpretation and explanation of the graph. ${ JSON . stringify ( self . dataContextForGraph ) } `
300
374
}
301
375
]
302
376
} ) ;
@@ -336,6 +410,11 @@ export const AssistantModel = types
336
410
if ( isImageSnapshotRequest ) {
337
411
self . uploadFileAfterRun = true ;
338
412
self . dataUri = res . values . exportDataUri ;
413
+ const graphID = resource . match ( / \[ ( \d + ) \] / ) ?. [ 1 ] ;
414
+ // We'll also send data for the attributes on the graph for additional context
415
+ self . dataContextForGraph = yield getGraphAttrData ( graphID ) ;
416
+ // TODO: Remove this console log when done with testing
417
+ console . log ( "Attribute data for graph" , self . dataContextForGraph ) ;
339
418
}
340
419
// remove any exportDataUri value that exists since it can be large and we don't need to send it to the assistant
341
420
res = isImageSnapshotRequest
0 commit comments