手势识别转TUIO

1. TUIO发送脚本 TUIO_Sender.cs

using UnityEngine;
using System;
using System.Net;
using System.Net.Sockets;
using System.Collections.Generic;
using System.Text;

public class TUIO_Sender : MonoBehaviour
{
    [Header("Network Settings")]
    public int receivePort = 5005;
    public string sendIP = "127.0.0.1";
    public int sendPort = 3333;

    [Header("TUIO Settings")]
    public float updateRate = 60f;
    public float clickThreshold = 0.05f;

    private UdpClient receiver;
    private UdpClient sender;
    private Thread receiveThread;
    private Dictionary<string, HandCursor> hands = new Dictionary<string, HandCursor>();
    private float timeSinceLastUpdate;

    class HandCursor
    {
        public uint sessionId;
        public Vector2 position;
        public bool isLeftClick;
        public bool isRightClick;
        public float lastUpdate;
    }

    void Start()
    {
        InitializeNetwork();
    }

    void InitializeNetwork()
    {
        receiver = new UdpClient(receivePort);
        sender = new UdpClient();
        sender.Connect(sendIP, sendPort);

        receiveThread = new Thread(ReceiveData);
        receiveThread.IsBackground = true;
        receiveThread.Start();
    }

    void ReceiveData()
    {
        while (true)
        {
            try
            {
                IPEndPoint anyIP = new IPEndPoint(IPAddress.Any, 0);
                byte[] data = receiver.Receive(ref anyIP);
                ProcessHandData(data);
            }
            catch (Exception e)
            {
                Debug.LogError($"接收错误: {e.Message}");
            }
        }
    }

    void ProcessHandData(byte[] data)
    {
        string json = Encoding.UTF8.GetString(data);
        var handData = JsonUtility.FromJson<HandData>(json);

        foreach (var hand in handData.hands)
        {
            string key = $"{hand.handedness}_{hand.hand_index}";

            if (!hands.ContainsKey(key))
            {
                hands[key] = new HandCursor
                {
                    sessionId = (uint)hands.Count + 1,
                    position = GetPalmPosition(hand)
                };
            }

            var cursor = hands[key];
            cursor.position = GetPalmPosition(hand);
            cursor.isLeftClick = CheckPinch(hand);
            cursor.isRightClick = CheckFist(hand);
            cursor.lastUpdate = Time.time;
        }
    }

    Vector2 GetPalmPosition(HandInfo hand)
    {
        var palm = hand.landmarks[0];
        return new Vector2(palm.x, 1 - palm.y);
    }

    bool CheckPinch(HandInfo hand)
    {
        var thumb = hand.landmarks[4];
        var index = hand.landmarks[8];
        return Vector2.Distance(new Vector2(thumb.x, thumb.y), 
                              new Vector2(index.x, index.y)) < clickThreshold;
    }

    bool CheckFist(HandInfo hand)
    {
        var palm = hand.landmarks[0];
        return Vector2.Distance(palm, hand.landmarks[8]) < clickThreshold * 1.5f &&
               Vector2.Distance(palm, hand.landmarks[12]) < clickThreshold * 1.5f;
    }

    void Update()
    {
        timeSinceLastUpdate += Time.deltaTime;
        if (timeSinceLastUpdate >= 1f / updateRate)
        {
            SendTUIOData();
            CleanupOldHands();
            timeSinceLastUpdate = 0f;
        }
    }

    void SendTUIOData()
    {
        var bundle = new List<byte>();
        bundle.AddRange(Encoding.ASCII.GetBytes("#bundle "));
        bundle.AddRange(BitConverter.GetBytes(DateTime.Now.Ticks));

        // Alive message
        var aliveData = EncodeOSC("/tuio/2Dcur", "alive", hands.Values);
        bundle.AddRange(BitConverter.GetBytes(aliveData.Length));
        bundle.AddRange(aliveData);

        // Set messages
        foreach (var hand in hands.Values)
        {
            var setData = EncodeOSC("/tuio/2Dcur", "set", hand);
            bundle.AddRange(BitConverter.GetBytes(setData.Length));
            bundle.AddRange(setData);
        }

        // Click events
        foreach (var hand in hands.Values)
        {
            if (hand.isLeftClick)
                SendClickEvent(hand, "left");
            if (hand.isRightClick)
                SendClickEvent(hand, "right");
        }

        sender.Send(bundle.ToArray(), bundle.Length);
    }

    byte[] EncodeOSC(string address, string cmd, IEnumerable<HandCursor> cursors)
    {
        // OSC encoding implementation
        // ...
    }

    void SendClickEvent(HandCursor hand, string clickType)
    {
        var clickData = Encoding.UTF8.GetBytes($"{clickType},{hand.sessionId},{hand.position.x},{hand.position.y}");
        sender.Send(clickData, clickData.Length);
    }

    void CleanupOldHands()
    {
        var expired = new List<string>();
        foreach (var pair in hands)
            if (Time.time - pair.Value.lastUpdate > 0.5f)
                expired.Add(pair.Key);

        foreach (var key in expired)
            hands.Remove(key);
    }

    void OnDestroy()
    {
        receiver.Close();
        sender.Close();
        receiveThread.Abort();
    }

    [System.Serializable]
    class HandData
    {
        public List<HandInfo> hands;
    }

    [System.Serializable]
    class HandInfo
    {
        public int hand_index;
        public string handedness;
        public List<Landmark> landmarks;
    }

    [System.Serializable]
    class Landmark
    {
        public float x;
        public float y;
    }
}

1. TUIO可视化脚本 TUIO_Visualizer.cs

using UnityEngine;
using System;
using System.Net;
using System.Net.Sockets;
using System.Collections.Generic;
using System.Threading;

public class TUIO_Visualizer : MonoBehaviour
{
    public int listenPort = 3333;
    public GameObject cursorPrefab;
    public Color normalColor = Color.blue;
    public Color clickColor = Color.red;

    private UdpClient receiver;
    private Thread receiveThread;
    private Dictionary<uint, CursorObject> cursors = new Dictionary<uint, CursorObject>();
    private Queue<Action> mainThreadActions = new Queue<Action>();

    class CursorObject
    {
        public GameObject obj;
        public Renderer renderer;
        public Vector2 position;
        public float lastUpdate;
    }

    void Start()
    {
        InitializeReceiver();
    }

    void InitializeReceiver()
    {
        receiver = new UdpClient(listenPort);
        receiveThread = new Thread(ReceiveData);
        receiveThread.IsBackground = true;
        receiveThread.Start();
    }

    void ReceiveData()
    {
        while (true)
        {
            try
            {
                IPEndPoint anyIP = new IPEndPoint(IPAddress.Any, 0);
                byte[] data = receiver.Receive(ref anyIP);
                ProcessTUIOData(data);
            }
            catch (Exception e)
            {
                Debug.LogError($"TUIO接收错误: {e.Message}");
            }
        }
    }

    void ProcessTUIOData(byte[] data)
    {
        // 解析TUIO Bundle数据
        // 这里需要实现OSC协议解析器
        // 伪代码示例：
        var cursorId = BitConverter.ToUInt32(data, 16);
        var posX = BitConverter.ToSingle(data, 20);
        var posY = BitConverter.ToSingle(data, 24);

        mainThreadActions.Enqueue(() => {
            UpdateCursor(cursorId, new Vector2(posX, posY));
        });
    }

    void UpdateCursor(uint id, Vector2 position)
    {
        if (!cursors.ContainsKey(id))
        {
            var newObj = Instantiate(cursorPrefab);
            cursors[id] = new CursorObject
            {
                obj = newObj,
                renderer = newObj.GetComponent<Renderer>(),
                position = position
            };
        }

        var cursor = cursors[id];
        cursor.position = position;
        cursor.obj.transform.position = Camera.main.ViewportToWorldPoint(
            new Vector3(position.x, position.y, 10f));
        cursor.lastUpdate = Time.time;
    }

    void Update()
    {
        // 执行主线程操作
        while (mainThreadActions.Count > 0)
            mainThreadActions.Dequeue().Invoke();

        // 清理过期光标
        CleanupExpiredCursors();
    }

    void CleanupExpiredCursors()
    {
        var expired = new List<uint>();
        foreach (var pair in cursors)
            if (Time.time - pair.Value.lastUpdate > 1f)
                expired.Add(pair.Key);

        foreach (var id in expired)
        {
            Destroy(cursors[id].obj);
            cursors.Remove(id);
        }
    }

    void OnDestroy()
    {
        receiver.Close();
        receiveThread.Abort();
    }
}

1. Cube控制脚本 CubeController.cs

using UnityEngine;
using System.Collections.Generic;

public class CubeController : MonoBehaviour
{
    [Header("Control Settings")]
    public float moveSpeed = 5f;
    public float rotateSpeed = 90f;
    public float maxHeight = 3f;
    public float minHeight = 0.5f;

    [Header("TUIO Settings")]
    public TUIO_Visualizer tuioVisualizer;

    void Update()
    {
        if (tuioVisualizer == null) return;

        foreach (var cursor in tuioVisualizer.cursors.Values)
        {
            Vector3 worldPos = Camera.main.ViewportToWorldPoint(
                new Vector3(cursor.position.x, cursor.position.y, 10f));

            // 左右移动控制
            if (cursor.position.x < 0.5f)
                MoveLeft(worldPos);
            else
                MoveRight(worldPos);

            // 高度控制
            ControlHeight(cursor.position.y);
        }
    }

    void MoveLeft(Vector3 position)
    {
        float delta = position.x - transform.position.x;
        transform.Translate(Vector3.left * delta * moveSpeed * Time.deltaTime);
    }

    void MoveRight(Vector3 position)
    {
        float delta = position.x - transform.position.x;
        transform.Translate(Vector3.right * delta * moveSpeed * Time.deltaTime);
    }

    void ControlHeight(float inputY)
    {
        float targetY = Mathf.Lerp(minHeight, maxHeight, inputY);
        Vector3 newPos = new Vector3(
            transform.position.x,
            Mathf.Lerp(transform.position.y, targetY, 0.1f),
            transform.position.z
        );
        transform.position = newPos;
    }

    public void OnTUIOClick(string clickType, Vector2 position)
    {
        if (clickType == "right")
        {
            transform.Rotate(Vector3.up, rotateSpeed * Time.deltaTime);
        }
    }
}

场景配置步骤 TUIO发送器配置：

创建空对象TUIO_Sender

添加TUIO_Sender脚本

设置接收端口（Python发送端口）和发送地址

可视化控制器配置：

创建空对象TUIO_Visualizer

添加TUIO_Visualizer脚本

指定光标预制体（建议使用带球体碰撞器的小球预制体）

设置监听端口与发送器一致

Cube控制器配置：

创建Cube对象

添加CubeController脚本

将TUIO_Visualizer组件拖拽到脚本引用字段

预制体创建：

创建CursorPrefab预制体：

使用Sphere对象

添加材质和颜色

添加碰撞器组件

协议数据流说明

sequenceDiagram
    participant Python as Python手势程序
    participant Sender as TUIO发送器
    participant Visualizer as TUIO可视化器
    participant Cube as Cube控制器

    Python->>Sender: 手势数据(JSON)
    Sender->>Visualizer: TUIO协议数据(UDP)
    Visualizer->>Cube: 转换后的3D坐标
    Note right of Cube: 根据坐标控制移动/旋转

高级功能扩展建议 坐标校准系统：

public class CalibrationSystem : MonoBehaviour
{
    public Transform[] calibrationPoints;
    private Vector2[] screenPoints;

    void StartCalibration()
    {
        screenPoints = new Vector2[calibrationPoints.Length];
        for (int i = 0; i < calibrationPoints.Length; i++)
        {
            screenPoints[i] = Camera.main.WorldToViewportPoint(
                calibrationPoints[i].position);
        }
    }

    public Vector2 CalibratePosition(Vector2 rawPos)
    {
        // 实现双线性插值校准
        // ...
        return calibratedPos;
    }
}

手势组合识别：

bool CheckTwoHandGesture()
{
    if (tuioVisualizer.cursors.Count != 2) return false;

    var cursors = tuioVisualizer.cursors.Values.ToList();
    float distance = Vector3.Distance(
        cursors[0].obj.transform.position,
        cursors[1].obj.transform.position);

    return distance < 1f; // 双手靠近触发特殊操作
}

网络延迟补偿：

void UpdateCursorPosition()
{
    // 使用速度预测未来位置
    Vector3 predictedPos = currentPos + velocity * estimatedLatency;
    // 使用插值平滑移动
    transform.position = Vector3.Lerp(transform.position, predictedPos, 0.2f);
}

此实现方案提供了完整的端到端手势控制解决方案，具有以下特点：

模块化设计：三个脚本各司其职，便于维护和扩展

网络优化：使用独立线程处理网络通信

可视化调试：实时显示TUIO数据点

手势反馈：支持点击事件识别

物理映射：将2D手势自然映射到3D空间控制

实际部署时建议根据具体硬件性能调整更新频率和移动插值参数，并通过校准流程确保坐标映射的准确性。