javascriptc#node.jsunity-game-enginewebrtc

Send text message from Unity Client to Javascript server using Unity Render Streaming and WebRTC


I am trying to setup a streaming app using Unity and currently Unity Render Streaming plugin is being used with the RTCPeerConnection to stream the video. I edited the bidirectional WebApp sample (sendvideo.js) to just send a video to the Unity app that is my client, where I am using the ReceiverSample.cs to visualize the video. I need to send back a text message could be "Hello world" for instance, from Unity to Javascript during the video streaming, but I wasn't successful to do it.

I am currently using the following:

And here is what I've tried to get the Unity side to send message to javascript webapp:

sendvideo.js

import * as Logger from "../../module/logger.js";

export class SendVideo {
  constructor(localVideoElement, remoteVideoElement) {
    this.localVideo = localVideoElement;
    this.remoteVideo = remoteVideoElement;
    this.peerConnection = null;
    this.dataChannel = null; // Add DataChannel variable
  }

  async startLocalVideo() {
    try {
      const videoElement = document.createElement('video');
      videoElement.src = '/videos/video.mp4'; // Path to your video file
      videoElement.muted = true;
      await videoElement.play();

      const stream = videoElement.captureStream();
      this.localVideo.srcObject = stream;
      await this.localVideo.play();

      this.initializePeerConnection();

    } catch (err) {
      Logger.error(`Error starting local video: ${err}`);
    }
  }

  // Set up WebRTC connection and DataChannel
  initializePeerConnection() {
    this.peerConnection = new RTCPeerConnection();

    this.dataChannel = this.peerConnection.createDataChannel("myDataChannel");

    this.dataChannel.onopen = () => {
      console.log("DataChannel open: Connection established with Unity client");
    };

    this.dataChannel.onmessage = (event) => {
      console.log(`Received message from Unity: ${event.data}`); 
    };

    this.peerConnection.createOffer().then(offer => {
      return this.peerConnection.setLocalDescription(offer);
    }).then(() => {
      console.log("Offer created and sent to Unity client");
    }).catch(err => console.error(err));
  }
}

And here is the ReceiverSample.cs code:

using System;
using System.Collections;
using UnityEngine;
using UnityEngine.UI;
using Unity.WebRTC; 

public class ReceiverSample : MonoBehaviour
{
#pragma warning disable 0649
    [SerializeField] private Button startButton;
    [SerializeField] private Button stopButton;
    [SerializeField] private InputField connectionIdInput;
    [SerializeField] private RawImage remoteVideoImage;
    [SerializeField] private ReceiveVideoViewer receiveVideoViewer;
    [SerializeField] private SingleConnection connection;
#pragma warning restore 0649

    private RTCDataChannel dataChannel;
    private float messageInterval = 2.0f; // Send a message every 2 seconds

    void Awake()
    {
        startButton.onClick.AddListener(OnStart);
        stopButton.onClick.AddListener(OnStop);
        if (connectionIdInput != null)
            connectionIdInput.onValueChanged.AddListener(input => connectionId = input);
        receiveVideoViewer.OnUpdateReceiveTexture += texture => remoteVideoImage.texture = texture;
    }

    private void OnStart()
    {
        if (string.IsNullOrEmpty(connectionId))
        {
            connectionId = System.Guid.NewGuid().ToString("N");
            connectionIdInput.text = connectionId;
        }
        connectionIdInput.interactable = false;

        // Create the connection
        connection.CreateConnection(connectionId, true);

        // Subscribe to the DataChannel event (available from the connection object)
        connection.OnDataChannel += OnDataChannelReceived;

        startButton.gameObject.SetActive(false);
        stopButton.gameObject.SetActive(true);
    }

    private void OnDataChannelReceived(RTCDataChannel channel)
    {
        dataChannel = channel;

        dataChannel.OnOpen += () =>
        {
            Debug.Log("DataChannel opened. Starting to send messages.");
            StartCoroutine(SendHelloWorldMessage());
        };

        dataChannel.OnClose += () =>
        {
            Debug.Log("DataChannel closed.");
        };

        dataChannel.OnMessage += OnMessageReceived;
    }

    private void OnMessageReceived(byte[] message)
    {
        string receivedMessage = System.Text.Encoding.UTF8.GetString(message);
        Debug.Log($"Message received from WebApp: {receivedMessage}");
    }

    private IEnumerator SendHelloWorldMessage()
    {
        while (dataChannel.ReadyState == RTCDataChannelState.Open)
        {
            dataChannel.Send("Hello World");
            Debug.Log("Sent 'Hello World' to JavaScript client");
            yield return new WaitForSeconds(messageInterval);
        }
    }

    private void OnStop()
    {
        StopAllCoroutines(); 
        connection.DeleteConnection(connectionId);
        connectionId = String.Empty;
        connectionIdInput.text = String.Empty;
        connectionIdInput.interactable = true;
        startButton.gameObject.SetActive(true);
        stopButton.gameObject.SetActive(false);
    }
}

The ReceiverSample.cs outputs the following errors:

Assets\Samples\Unity Render Streaming\3.0.1-preview\Example\Receiver\ReceiverSample.cs(15,30): error CS0246: The type or namespace name 'SingleConnection' could not be found (are you missing a using directive or an assembly reference?) Assets\Samples\Unity Render Streaming\3.0.1-preview\Example\Receiver\ReceiverSample.cs(15,30): error CS0246: The type or namespace name 'SingleConnection' could not be found (are you missing a using directive or an assembly reference?)

I'd like to know if this is the correct approach or is my code approach using RTCDataChannel will work for my usecase, and if so, how can I fix it to get it working the way I need it to work.


Solution

  • The best solution for my case was to work with UDP:

    udpClient = new UdpClient();
    endPoint = new IPEndPoint(IPAddress.Parse("127.0.0.1"), 99999); 
    byte[] data = Encoding.UTF8.GetBytes(dataToSend);
    udpClient.Send(data, data.Length, endPoint);
    Debug.Log("Sent data to JavaScript server: " + dataToSend);