All files / web/src/lib/voice toolCallHelpers.ts

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
/**
 * Helper functions for sending messages on the OpenAI Realtime data channel.
 */

/** Send a tool call response (function_call_output) and optionally prompt a model response. */
export function sendToolResponse(
  dc: RTCDataChannel,
  callId: string,
  output: unknown,
  promptResponse = true
) {
  dc.send(
    JSON.stringify({
      type: 'conversation.item.create',
      item: {
        type: 'function_call_output',
        call_id: callId,
        output: JSON.stringify(output),
      },
    })
  )
  if (promptResponse) {
    dc.send(JSON.stringify({ type: 'response.create' }))
  }
}

/** Send a system-level text message and optionally prompt a model response. */
export function sendSystemMessage(dc: RTCDataChannel, text: string, promptResponse = false) {
  dc.send(
    JSON.stringify({
      type: 'conversation.item.create',
      item: {
        type: 'message',
        role: 'user',
        content: [{ type: 'input_text', text }],
      },
    })
  )
  if (promptResponse) {
    dc.send(JSON.stringify({ type: 'response.create' }))
  }
}

/**
 * Send a combined text + optional image context update as a single conversation item.
 * More efficient than separate text + image messages — uses one item in the context window.
 */
export function sendContextUpdate(
  dc: RTCDataChannel,
  text: string,
  base64DataUrl?: string | null,
  promptResponse = false
) {
  console.log(
    '[voice-helpers] sendContextUpdate: textLen=%d, hasImage=%s, promptResponse=%s, dc.readyState=%s',
    text.length,
    !!base64DataUrl,
    promptResponse,
    dc.readyState
  )
  const content: Array<Record<string, string>> = [{ type: 'input_text', text }]
  if (base64DataUrl) {
    // OpenAI Realtime API expects image_url as a full data URI
    const dataUri = base64DataUrl.includes(',')
      ? base64DataUrl
      : `data:image/png;base64,${base64DataUrl}`
    content.push({ type: 'input_image', image_url: dataUri })
  }
  dc.send(
    JSON.stringify({
      type: 'conversation.item.create',
      item: { type: 'message', role: 'user', content },
    })
  )
  if (promptResponse) {
    dc.send(JSON.stringify({ type: 'response.create' }))
  }
}

/** Send a user text message to the voice session and prompt a model response. */
export function sendUserText(dc: RTCDataChannel, text: string) {
  console.log(
    '[voice-helpers] sendUserText: textLen=%d, dc.readyState=%s',
    text.length,
    dc.readyState
  )
  dc.send(
    JSON.stringify({
      type: 'conversation.item.create',
      item: {
        type: 'message',
        role: 'user',
        content: [{ type: 'input_text', text }],
      },
    })
  )
  dc.send(JSON.stringify({ type: 'response.create' }))
}

/**
 * Send an image to the conversation via input_image content part.
 * Used for mid-conversation visual context (e.g. construction screenshots).
 */
export function sendImageContext(
  dc: RTCDataChannel,
  base64DataUrl: string,
  promptResponse = false
) {
  // OpenAI Realtime API expects image_url as a full data URI
  const dataUri = base64DataUrl.includes(',')
    ? base64DataUrl
    : `data:image/png;base64,${base64DataUrl}`

  dc.send(
    JSON.stringify({
      type: 'conversation.item.create',
      item: {
        type: 'message',
        role: 'user',
        content: [
          {
            type: 'input_image',
            image_url: dataUri,
          },
        ],
      },
    })
  )
  if (promptResponse) {
    dc.send(JSON.stringify({ type: 'response.create' }))
  }
}