Is the WebRTC-based Realtime API still not supported in Go language?

API Bugs
, ,
1

I am currently using the Go program below for development and debugging. There are no errors, the connection is successful, and messages can be sent successfully, but I am unable to receive a response to the messages, not even an error message.

package main

import (
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/geek/client/backend/utils"
	"github.com/pion/webrtc/v4"
)

func main() {

	peerConnection, _ := webrtc.NewPeerConnection(webrtc.Configuration{
		ICEServers: []webrtc.ICEServer{
			{URLs: []string{"stun:stun.l.google.com:19302"}},
		},
	})

	_, response, _ := utils.RealtimeSessions("sk-proj-6************************************")
	fmt.Println("[assistant] RealtimeSessions response:", response)
	if response.ClientSecret.Value == "" {
		fmt.Println("[assistant] RealtimeSessions ephemeralKey is empty, cannot proceed")
		return
	}

	dataChannel, err := peerConnection.CreateDataChannel("oai-events", nil)
	if err != nil {
		fmt.Println("[assistant] Failed to create data channel:", err)
		return
	}

	dataChannel.OnOpen(func() {
		fmt.Println("[assistant] DataChannel 'oai-events' open!")
		requestJSON := `{"type": "session.update","session": {"modalities": ["text"]}}`
		sendErr := dataChannel.SendText(requestJSON)
		if sendErr != nil {
			fmt.Printf("[assistant] Error sending JSON: %s\n", sendErr)
		}
		go func() {
			time.Sleep(5 * time.Second)
			requestJSON := `{"type": "response.create","response": {"modalities": ["text"],"instructions": "Write a haiku about code"}}`
			sendErr := dataChannel.SendText(requestJSON)
			if sendErr != nil {
				fmt.Printf("[assistant] Error sending JSON: %s\n", sendErr)
			}
		}()
	})

	dataChannel.OnMessage(func(msg webrtc.DataChannelMessage) {
		fmt.Println("[assistant] got message from OpenAI:", string(msg.Data))
	})

	dataChannel.OnError(func(err error) {
		fmt.Println("[assistant] DataChannel 'oai-events' error:", err)
	})

	dataChannel.OnClose(func() {
		fmt.Println("[assistant] DataChannel 'oai-events' closed!")
	})

	peerConnection.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
		fmt.Println("[assistant] got track from OpenAI:", track.ID())
	})

	peerConnection.OnICEConnectionStateChange(func(state webrtc.ICEConnectionState) {
		fmt.Println("[assistant] ICE state changed:", state.String())
	})

	offer, err := peerConnection.CreateOffer(nil)
	if err != nil {
		fmt.Println("[assistant] Failed to create offer:", err)
		return
	}

	fmt.Println("[assistant] Offer SDP:\n", offer.SDP)

	err = peerConnection.SetLocalDescription(offer)
	if err != nil {
		fmt.Println("[assistant] Failed to set local description:", err)
		return
	}

	baseUrl := "https://api.openai.com/v1/realtime"
	req, _ := http.NewRequest("POST", baseUrl+"?model=gpt-4o-realtime-preview-2024-12-17", strings.NewReader(offer.SDP))
	req.Header.Set("Authorization", "Bearer "+response.ClientSecret.Value)
	req.Header.Set("Content-Type", "application/sdp")
	req.Header.Set("User-Agent", `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36`)

	client := &http.Client{}

	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("[assistant] Failed to send offer:", err)
		return
	}
	defer resp.Body.Close()

	answerSdpBytes, _ := io.ReadAll(resp.Body)
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		fmt.Println("[assistant] Failed to get answer:", string(answerSdpBytes))
		return
	}

	fmt.Println("[assistant] Answer SDP:\n", string(answerSdpBytes))

	answer := webrtc.SessionDescription{
		Type: webrtc.SDPTypeAnswer,
		SDP:  string(answerSdpBytes),
	}

	err = peerConnection.SetRemoteDescription(answer)
	if err != nil {
		fmt.Println("[assistant] Failed to set remote description:", err)
		return
	}

	fmt.Println("[assistant] Ready to connect to OpenAI Realtime API!")

	select {}
}

Log:

[root@robot ~/develop/backend]# go run text.go
[assistant] RealtimeSessions response: {{ } sess_AijGD3P*******d realtime.session gpt-4o-realtime-preview-2024-12-17 You are a friendly assistant. {ek_676d69e14*********ee542b}}
[assistant] Offer SDP:
 v=0
o=- 2781977947604390403 1735223777 IN IP4 0.0.0.0
s=-
t=0 0
a=msid-semantic:WMS*
a=fingerprint:sha-256 7D:46:22:BD:5E:7F:6D:FA:4A:5D:02:30:D6:A1:DB:39:01:80:32:40:B1:6D:CF:C8:3F:2E:9A:8D:30:06:2C:F9
a=extmap-allow-mixed
a=group:BUNDLE 0
m=application 9 UDP/DTLS/SCTP webrtc-datachannel
c=IN IP4 0.0.0.0
a=setup:actpass
a=mid:0
a=sendrecv
a=sctp-port:5000
a=ice-ufrag:qCVlKsptGesQpzPx
a=ice-pwd:uMlDaOvteyxCYKwCDJUDdcICLHCjohOS

[assistant] Answer SDP:
 v=0
o=- 5984563547975346745 1735223777 IN IP4 0.0.0.0
s=-
t=0 0
a=msid-semantic:WMS*
a=fingerprint:sha-256 0E:F6:3B:CC:B3:6F:4E:02:9E:BE:B5:BE:DB:FA:12:81:E8:9E:66:D6:13:0C:57:73:C4:A6:DC:3F:21:E4:24:80
a=extmap-allow-mixed
a=group:BUNDLE 0
m=application 9 UDP/DTLS/SCTP webrtc-datachannel
c=IN IP4 0.0.0.0
a=setup:active
a=mid:0
a=sendrecv
a=sctp-port:5000
a=ice-ufrag:UWDlgWYcOzCvvaYB
a=ice-pwd:rzPmkofEfvAKlajChqANfBcdYwKRkcjH
a=candidate:4146524507 1 udp 2130706431 40.84.168.13 3478 typ host
a=candidate:4146524507 2 udp 2130706431 40.84.168.13 3478 typ host
a=end-of-candidates

[assistant] Ready to connect to OpenAI Realtime API!
[assistant] ICE state changed: checking
[assistant] ICE state changed: connected
[assistant] DataChannel 'oai-events' open!

I hope to receive your assistance. Thank you very much.

2024-12-27

2

Hey @FrostByte sorry I missed this!

Pion is supported. Check out GitHub - realtime-ai/openai-realtime-webrtc-go: openai webrtc go

Looking at your code I think you need to do a AddTrack or AddTransceiverFromKind before you create your offer.

If you want to put code up on GitHub I would love to help!