Hello, I’m very new to Unity in general.
I’m working on a project that requires me to connect a IR camera that tracks upper body joints. The camera sends joint data from a small custom application written in C++.
I’ve successfully connected the camera and made Game Objects in the scene move by transforming their coordinates, but I noticed that they are not in sync with the camera’s tracking.
Now, packets from the C++ app are being sent continuously, and I’m receiving every packet successfully, but they are around 5 seconds late after the game has been running for couple of seconds.
I’m not sure what I should do to sync them up. I do not need every single packet to be received, because many packets contain identical data, but I want them to be displayed in the game in (or close to) realtime.
Here’s what it looks like when debugging:[122340-screenshot-1.png*_|122340]
Code for “server socket” that receives data from the C++ app that camera sends to it.
using System;
using System.Net;
using System.Net.Sockets;
using System.Text;
using UnityEngine;
public class TcpSocket
{
private string ipAddress;
private int port;
private Socket socket, clientSocket;
private byte[] buffer = new byte[1024];
private long counter = 0;
private int backlog = 10;
public delegate void OnMessageReceived(string message, long counter);
public event OnMessageReceived MessageReceived;
public TcpSocket(string ipAddress, int port)
{
this.ipAddress = ipAddress;
this.port = port;
socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
#region added for server emulation
socket.Bind(new IPEndPoint(IPAddress.Any, port));
Debug.Log("Bound socket to port " + port);
socket.Listen(backlog);
Debug.Log("Started listening...");
Accept();
#endregion
}
#region MethodsForServerEmulation
private void AcceptedCallback(IAsyncResult result)
{
Console.WriteLine("Accept callback called... ");
clientSocket = socket.EndAccept(result);
if (socket.Connected) { Console.WriteLine("A client has connected... "); }
clientSocket.BeginReceive(buffer, 0, buffer.Length, SocketFlags.None, ReceivedCallback, clientSocket);
Accept();
}
public void Accept()
{
socket.BeginAccept(AcceptedCallback, null);
Debug.Log("Beginning accept... ");
}
#endregion
public IAsyncResult Connect()
{
return socket.BeginConnect(new IPEndPoint(IPAddress.Parse(ipAddress), port), ConnectCallback, null);
}
private void ConnectCallback(IAsyncResult result)
{
if (socket.Connected)
{
Debug.Log("Connected to server!");
socket.BeginReceive(buffer, 0, buffer.Length, SocketFlags.None, ReceivedCallback, null);
}
}
private void ReceivedCallback(IAsyncResult result)
{
clientSocket = result.AsyncState as Socket;
Debug.Log("Entered Receive callback...");
int bufferLength = socket.EndReceive(result);
if (bufferLength > 0)
{
counter++;
string message = Encoding.ASCII.GetString(buffer, 0, buffer.Length);
if (MessageReceived != null) { MessageReceived(message, counter); }
// Handle packet
Array.Clear(buffer, 0, buffer.Length);
clientSocket.BeginReceive(buffer, 0, buffer.Length, SocketFlags.None, ReceivedCallback, clientSocket);
}
else { Debug.Log("Nothing received from socket"); }
}
}
And Controller script, that uses that data to move game objects in the scene:
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System;
public class RealsenseController : MonoBehaviour
{
private TcpSocket clientSocket;
#region gameObjects
public GameObject leftHand;
public GameObject rightHand;
public GameObject leftShoulder;
public GameObject rightShoulder;
public GameObject head;
public GameObject middleBody;
#endregion
private float coordZ = -9f;
Vector3[] gameobjectVectors = new Vector3[8];
private bool IsMessageReceived = false;
// Use this for initialization
void Awake()
{
clientSocket = new TcpSocket("127.0.0.1", 54000);
clientSocket.MessageReceived += ClientSocket_MessageReceived;
//clientSocket.Connect();
// Initializing vectors that will be changed through coordinates we'll be receiving
#region gameobjectVectors initialization
gameobjectVectors[0] = new Vector3(0f, 0f, coordZ); //0 - leftShoulder
gameobjectVectors[1] = new Vector3(0f, 0f, coordZ); //1 - rightShoulder
gameobjectVectors[2] = new Vector3(0f, 0f, coordZ); //2 - leftHand
gameobjectVectors[3] = new Vector3(0f, 0f, coordZ); //3 - rightHand
gameobjectVectors[4] = new Vector3(0f, 0f, coordZ); //4 - head
gameobjectVectors[5] = new Vector3(0f, 0f, coordZ); //5 - midBody
#endregion
}
private void ClientSocket_MessageReceived(string message, long counter)
{
//IsMessageReceived = true;
ReformatMessage(message);
Debug.Log(message);
Debug.Log(DateTime.Now.Hour + ":" + DateTime.Now.Minute + ":" + DateTime.Now.Second);
}
// Update is called once per frame
void FixedUpdate()
{
//if (IsMessageReceived == true)
//{
leftShoulder.transform.position = gameobjectVectors[0];
rightShoulder.transform.position = gameobjectVectors[1];
leftHand.transform.position = gameobjectVectors[2];
rightHand.transform.position = gameobjectVectors[3];
head.transform.position = gameobjectVectors[4];
middleBody.transform.position = gameobjectVectors[5];
//IsMessageReceived = false;
//}
}
private void ReformatMessage(string message)
{
float x, y;
string[] perBodyparts = message.Split(' ');
string[][] wholeMessage = new string[perBodyparts.Length - 1][];
for (int i = 0; i < perBodyparts.Length - 1; i++) // Splitting message into parts, and changing vector coordinates
{
wholeMessage _= perBodyparts*.Split(';');*_
if(i <= 5)
{
x = float.Parse(wholeMessage*[1]);*
x = (x / 100f) + 3f;
y = float.Parse(wholeMessage*[2]);*
y = 7f - (y / 100f);
Debug.Log(wholeMessage*[0] + “;” + x + “;” + y + “;”);*
gameobjectVectors*.x = x;*
gameobjectVectors*.y = y;*
}
else
{
Debug.Log(wholeMessage*[0]);*
}
}
}
}
*
Any ideas what I should change?
Thanks!_