使用 c# unity解码 GStreamer jpeg 流

Question

我有一个流，我计划用 GStreamer 编码并通过 TCP 发送，然后在 Unity 中对其进行解码。

这是我的 GStreamer 管道：

gst-launch-1.0 v4l2src ! queue ! videoconvert ! video/x-raw,width=640,height=480,framerate=30/1 ! jpegenc ! multipartmux ! tcpserversink host=127.0.0.1 port=5000

我在获取每个 jpeg 帧的 字节数组大小 时遇到问题。 为了正确读取每一帧，我需要首先获取每一帧的大小，并创建一个该大小的字节数组。 var read = serverStream.Read(imageBytes, 0, **size**);

如果我从管道中取出multipartmux，它似乎是一长串数据，并在收到第一帧后停止（serverStream.DataAvailable = FALSE）

如果我在管道中保留multipartmux，我将首先收到一个长度约为70~80的字节数组，然后我得到数据序列。 看起来像这样：

 { 45, 45, 84, 104, 105, 115, 82, 97, 110, 100, 111, 109, 83, 116, 114, 105, 110, 103, 13, 10, 67, 111, 110, 116, 101, 110, 116, 45, 84, 121, 112, 101, 58, 32, 105, 109, 97, 103, 101, 47, 106, 112, 101, 103, 13, 10, 67, 111, 110, 116, 101, 110, 116, 45, 76, 101, 110, 103, 116, 104, 58, 32, 51, 54, 55, 49, 52, 13, 10, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,}

但我不明白那个 Byte 数组的含义...是 Byte 数组的大小以某种方式描述的吗？

如果有人可以帮助我解释这些数据，或者给我一些关于 GStreamer jpeg 编码以及如何解码它们的见解，那就太好了。谢谢！

 private void readFrameByteArray(int size) {
            bool disconnected = false;
            UnityEngine.Debug.Log("- image size: " + size);
            NetworkStream serverStream = client.GetStream();
            serverStream.ReadTimeout = 10;
            UnityEngine.Debug.Log("ckp 1");
            byte[] imageBytes = new byte[size];
            var total = 0;
            var sb = new StringBuilder("imageBytes[] { ");
            do
            {
            //serverStream.ReadTimeout = 100;
            UnityEngine.Debug.Log("ckp2: "+ serverStream.DataAvailable);
                if (!serverStream.DataAvailable)return;
            
            var read = serverStream.Read(imageBytes, total, size);
            UnityEngine.Debug.Log("ckp3");
            if (read == 0)
                {
                    disconnected = true;
                    break;
                }
                total += read;
                //UnityEngine.Debug.Log("- read: " + read);
                UnityEngine.Debug.Log("- FrameByteArray total: " + total);
                foreach (var b in imageBytes)
                {
                    sb.Append(b + ", ");
                    //if (b == 0) break;
                }
                sb.Append("}");
                UnityEngine.Debug.Log(sb.ToString());
            } while (serverStream.DataAvailable);
            UnityEngine.Debug.Log("!serverStream.DataAvailable");
            

            bool readyToReadAgain = false;

            //Display Image
            if (!disconnected) {
                //Display Image on the main Thread
                Loom.QueueOnMainThread(() => {
                    loadReceivedImage(imageBytes);
                    readyToReadAgain = true;
                });
            }

            //Wait until old Image is displayed
            while (!readyToReadAgain) {
                UnityEngine.Debug.Log("!readyToReadAgain");
                System.Threading.Thread.Sleep(1);
            }
        }

Answer 1

对于任何正在寻找解决方案的人，这就是我最终得到的方式：

 public class _TextureReceiver : MonoBehaviour
    {
        public int port = 5000;
        public string IP = "127.0.0.1";
        TcpClient client;
        private NetworkStream serverStream;

        [HideInInspector]
        public Texture2D texture;

        private bool stop = false;

        [Header("Must be the same in sender and receiver")]
        public int messageByteLength = 24;

        public int maxFrameSize = 40000;

        int try_count = 1;

        // Use this for initialization
        void Start()
        {
            //UnityEngine.Debug.Log("Start!");
            Application.runInBackground = true;

            client = new TcpClient();

            //Connect to server from another Thread
            Loom.RunAsync(() =>
            {
                TCP_Connect();

            });
        }

        public void enStop()
        {
            stop = true;

            if (client != null)
            {
                client.Close();
            }

            if (serverStream != null)
            {
                serverStream.Close();
            }
            return;
        }

        public void TCP_Connect()
        {
            client = new TcpClient();
            IAsyncResult result = client.BeginConnect(IPAddress.Parse(IP), port, null, null);
            result.AsyncWaitHandle.WaitOne(1000, true);
            if (!client.Connected)
            {
                if (try_count > 0)
                {
                   // UnityEngine.Debug.LogError("Ethernet Connection Error, Retry");
                    try_count++;
                    client.Close();
                    TCP_Connect();
                }
                else
                {
                    client.Close();
                    //UnityEngine.Debug.Log("Missing Connection");
                }
            }
            else
            {
                //UnityEngine.Debug.Log("V");
                imageReceiver();
            }
        }
        public void imageReceiver()
        {
            //While loop in another Thread is fine so we don't block main Unity Thread
            Loom.RunAsync(() =>
            {
                while (!stop)
                {
                    readFrameByteArray(maxFrameSize);
                }
            });
        }


        private void readFrameByteArray(int size)
        {
            bool disconnected = false;
            serverStream = client.GetStream();

            byte[] imageBytes = new byte[size];
            var total = 0;

            //UnityEngine.Debug.Log("serverStream.DataAvailable: "+ serverStream.DataAvailable);
            if (!serverStream.DataAvailable) return;

            var read = serverStream.Read(imageBytes, total, size);
            if (read == 0)
            {
                disconnected = true;
                //break;
            }
            total += read;
            if (read == maxFrameSize) maxFrameSize += 5000;
            //UnityEngine.Debug.Log("[V]read: " + read);


            bool readyToReadAgain = false;

            //Display Image
            if (!disconnected)
            {
                //Display Image on the main Thread
                Loom.QueueOnMainThread(() =>
                {
                    loadReceivedImage(imageBytes);
                    readyToReadAgain = true;

                });
            }

            //Wait until old Image is displayed
            while (!readyToReadAgain)
            {
                System.Threading.Thread.Sleep(1);
            }

            return;
        }


        void loadReceivedImage(byte[] receivedImageBytes)
        {
            //UnityEngine.Debug.Log("* load Received Image *");

            if (texture)
            {
                if (receivedImageBytes[0] == 255 && receivedImageBytes[1] == 216 && receivedImageBytes[2] == 255 && receivedImageBytes[3] == 224 && receivedImageBytes[4] == 0 && receivedImageBytes[5] == 16 && receivedImageBytes[6] == 74 && receivedImageBytes[7] == 70 && receivedImageBytes[8] == 73 && receivedImageBytes[9] == 70)
                {
                    //UnityEngine.Debug.Log("* texture *");
                    texture.LoadImage(receivedImageBytes);
                }

            }
        }

        public void SetTargetTexture(Texture2D t)
        {
            texture = t;
        }

        public void OnApplicationQuit()
        {
            enStop();
        }




    }