Pass OpenCV coordinates to Unity

Hello!

I would like to pass x and y coordinates from OpenCV with UDP to Unity and use them in the world space. The idea is to capture movement and send the coordinates to a 3D model in Unity to move its arm. How can I adapt the coordinates that the arm is not stretched or look in any direction? I have a 400 x 400 frame. So if I get for x = 220 and for y = 180 I would like that the arm in Unity is looking in this position or transform him to it. I tested localPosition and Position in Unity and it did not worked, also ScreenToWorldPoint does not fit my final goal, does it?
You can download a video where I shwo the problem: https://cloudlogin02.world4you.com/index.php/s/oQe4dyUGGKAbIiJ

I am happy for some suggestions!

Regards

Some code for OpenCV and tracking the contours:

for c in ex_cnts:
    M = cv2.moments(c)
    (x,y,w,h) = cv2.boundingRect(c)
    if w > 20 and h > 20 and w < 100 and h < 100:
       cv2.rectangle(picture, (x,y), (x+w,y+h), (255, 0, 0), 2)
       ex_center_x = int(M['m10']/M['m00'])
       ex_center_y = int(M['m01']/M['m00'])
       if ex_center_x > center_x:
            leftx = x
            lefty = y
            print "in for " + str(y)
            messagel = "leftx " + str(leftx)
            messager = "lefty " + str(lefty)
            print messager
            # # sending data to Unity
            sock.sendto(messagel , (UDP_IP, UDP_PORT))
            sock.sendto(messager , (UDP_IP, UDP_PORT))
            left = (ex_center_x, ex_center_y)
            cv2.circle(picture, (ex_center_x, ex_center_y), 7, (255, 255, 255), -1)
            cv2.putText(picture, "left", (ex_center_x - 20, ex_center_y - 20),
                cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)
       if ex_center_x < center_x: 
            leftx = x
            lefty = y
            messagel = "leftx " + str(leftx)
            messager = "lefty " + str(lefty)
            sock.sendto(messagel , (UDP_IP, UDP_PORT))
            sock.sendto(messager , (UDP_IP, UDP_PORT))
            right = (ex_center_x, ex_center_y)
            cv2.circle(picture, (ex_center_x, ex_center_y), 7, (255, 255, 255), -1)
            cv2.putText(picture, "right", (ex_center_x - 20, ex_center_y - 20),
                cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)

And the script I use in Unity:

using UnityEngine;
using System;
using System.Collections;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;

public class SocketCLient : MonoBehaviour {

	// Use this for initialization

	public GameObject lefthand;
	public GameObject righthand;
	public GameObject body;
	private float xPos = 10.0f;
	private float yPos = 10.0f;
	private float bxPos = 0;
	private float byPos = 0;
	private String newString;

	Thread receiveThread;
	UdpClient client;
	public int port;

	//info

	public string lastReceivedUDPPacket = "";
	public string allReceivedUDPPackets = "";

	void Start () {
		init();
	}

	void OnGUI(){
		Rect  rectObj=new Rect (40,10,200,400);
		
		GUIStyle  style  = new GUIStyle ();
		
		style .alignment  = TextAnchor.UpperLeft;
		
		GUI .Box (rectObj,"# UDPReceive

127.0.0.1 “+port +” #
"

		          //+ "shell> nc -u 127.0.0.1 : "+port +" 

"

		          + "

Last Packet:
"+ lastReceivedUDPPacket

		          //+ "

All Messages:
"+allReceivedUDPPackets

		          ,style );

	}

	private void init(){
		print ("UPDSend.init()");

		port = 5065;

		print ("Sending to 127.0.0.1 : " + port);

		receiveThread = new Thread (new ThreadStart(ReceiveData));
		receiveThread.IsBackground = true;
		receiveThread.Start ();

	}

	private void ReceiveData(){
		client = new UdpClient (port);
		while (true) {
			try{
				IPEndPoint anyIP = new IPEndPoint(IPAddress.Parse("127.0.0.1"), port);
				byte[] data = client.Receive(ref anyIP);

				string text = Encoding.UTF8.GetString(data);
				print (">> " + text);
				lastReceivedUDPPacket=text;
				allReceivedUDPPackets=allReceivedUDPPackets+text;
				if (text.Contains("leftx")){
					newString = text.Substring(5, 4);
					xPos = int.Parse(newString);
				}
				if (text.Contains("lefty")){
					newString = text.Substring(5, 4);
					yPos = int.Parse(newString);
					// if (yPos > 140)
					// 	yPos = -yPos;
				}
				if (text.Contains("bodyx")){
					newString = text.Substring(5, 4);
					bxPos = int.Parse(newString);
				}
				if (text.Contains("bodyy")){
					newString = text.Substring(5, 4);
					byPos = int.Parse(newString);
				}
				//xPos = int.Parse(text);
				//xPos *= 0.021818f;
			}catch(Exception e){
				print (e.ToString());
			}
		}
	}

	public string getLatestUDPPacket(){
		allReceivedUDPPackets = "";
		return lastReceivedUDPPacket;
	}
	
	// Update is called once per frame
	void Update () {

		lefthand.transform.position = new Vector3(0, xPos , yPos);
		body.transform.position = new Vector3(bxPos, byPos, 0);
		// if(yPos < byPos)
		// 	lefthand.transform.position = new Vector3(xPos, yPos , 0);
		// if(yPos > byPos)
		// 	lefthand.transform.position = new Vector3(xPos, yPos , 0);	
		//lefthand.transform.Rotate(0, yPos,  -10, Space.World);
	}

	void OnApplicationQuit(){
		if (receiveThread != null) {
			receiveThread.Abort();
			Debug.Log(receiveThread.IsAlive); //must be false
		}
	}
}

Edit:

I also tried to only rotate the y- axis of the arm. Maybe the script is wrong?

targetPoint = new Vector3(target.transform.position.x, transform.position.y, target.transform.position.z) - transform.position;
targetRotation = Quaternion.LookRotation (-targetPoint, Vector3.up);
transform.rotation = Quaternion.Slerp(transform.rotation, targetRotation, Time.deltaTime * 2.0f);

Have you solved this problem? If yes, can you please help me?

Quizás este proyecto sirva GitHub - hasanavi/OpenCV-Unity3D-Object-Tracking: OpenCV Object Tracking using CamShift algorithm and Unity3d Mashup