将 Kinect for Windows v2 的深度和颜色流保存为图像文件（例如 png 或 jpg）

Question

我正在尝试编写一个应用程序，将 Kinect for Windows v2 的深度和颜色流保存为图像文件（如 png 或 jpg）。因此，我使用了 Kinect SDK v2 示例（因为我之前没有使用 C# 或 Kinect API 的经验）。我修改了 ColorBasics-WPF 示例代码以实现我的目标。这是只将颜色流转换为png文件的代码（我修改的唯一部分是Reader_ColorFrameArrived函数）：

//------------------------------------------------------------------------------
// <copyright file="MainWindow.xaml.cs" company="Microsoft">
//     Copyright (c) Microsoft Corporation.  All rights reserved.
// </copyright>
//------------------------------------------------------------------------------
namespace Microsoft.Samples.Kinect.ColorBasics
{
    using System;
    using System.ComponentModel;
    using System.Diagnostics;
    using System.Globalization;
    using System.IO;
    using System.Windows;
    using System.Windows.Media;
    using System.Windows.Media.Imaging;
    using Microsoft.Kinect;
    using System.Collections.Generic;


    /// <summary>
    /// Interaction logic for MainWindow
    /// </summary>
    public partial class MainWindow : Window, INotifyPropertyChanged
    {
        static int count = 0;

        /// <summary>
        /// Active Kinect sensor
        /// </summary>
        private KinectSensor kinectSensor = null;

        /// <summary>
        /// Reader for color frames
        /// </summary>
        private ColorFrameReader colorFrameReader = null;

        /// <summary>
        /// Bitmap to display
        /// </summary>
        private WriteableBitmap colorBitmap = null;

        /// <summary>
        /// Current status text to display
        /// </summary>
        private string statusText = null;

        /// <summary>
        /// Initializes a new instance of the MainWindow class.
        /// </summary>
        public MainWindow()
        {
            // get the kinectSensor object
            this.kinectSensor = KinectSensor.GetDefault();

            // open the reader for the color frames
            this.colorFrameReader = this.kinectSensor.ColorFrameSource.OpenReader();

            // wire handler for frame arrival
            this.colorFrameReader.FrameArrived += this.Reader_ColorFrameArrived;

            // create the colorFrameDescription from the ColorFrameSource using Bgra format
            FrameDescription colorFrameDescription = this.kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Bgra);

            // create the bitmap to display
            this.colorBitmap = new WriteableBitmap(colorFrameDescription.Width, colorFrameDescription.Height, 96.0, 96.0, PixelFormats.Bgr32, null);

            // set IsAvailableChanged event notifier
            this.kinectSensor.IsAvailableChanged += this.Sensor_IsAvailableChanged;

            // open the sensor
            this.kinectSensor.Open();

            // set the status text
            this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
                                                            : Properties.Resources.NoSensorStatusText;

            // use the window object as the view model in this simple example
            this.DataContext = this;

            // initialize the components (controls) of the window
            this.InitializeComponent();
        }

        /// <summary>
        /// INotifyPropertyChangedPropertyChanged event to allow window controls to bind to changeable data
        /// </summary>
        public event PropertyChangedEventHandler PropertyChanged;

        /// <summary>
        /// Gets the bitmap to display
        /// </summary>
        public ImageSource ImageSource
        {
            get
            {
                return this.colorBitmap;
            }
        }

        /// <summary>
        /// Gets or sets the current status text to display
        /// </summary>
        public string StatusText
        {
            get
            {
                return this.statusText;
            }

            set
            {
                if (this.statusText != value)
                {
                    this.statusText = value;

                    // notify any bound elements that the text has changed
                    if (this.PropertyChanged != null)
                    {
                        this.PropertyChanged(this, new PropertyChangedEventArgs("StatusText"));
                    }
                }
            }
        }

        /// <summary>
        /// Execute shutdown tasks
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void MainWindow_Closing(object sender, CancelEventArgs e)
        {
            if (this.colorFrameReader != null)
            {
                // ColorFrameReder is IDisposable
                this.colorFrameReader.Dispose();
                this.colorFrameReader = null;
            }

            if (this.kinectSensor != null)
            {
                this.kinectSensor.Close();
                this.kinectSensor = null;
            }
        }

        /// <summary>
        /// Handles the user clicking on the screenshot button
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void ScreenshotButton_Click(object sender, RoutedEventArgs e)
        {
            if (this.colorBitmap != null)
            {
                // create a png bitmap encoder which knows how to save a .png file
                BitmapEncoder encoder = new PngBitmapEncoder();

                // create frame from the writable bitmap and add to encoder
                encoder.Frames.Add(BitmapFrame.Create(this.colorBitmap));
                //bitmaps.Add(BitmapFrame.Create(this.colorBitmap.Clone()));

                string time = count.ToString();
                //string time = System.DateTime.Now.ToString("hh'-'mm'-'ss", CultureInfo.CurrentUICulture.DateTimeFormat);

                string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);

                string path = Path.Combine(myPhotos, "KinectScreenshot-Color-" + time + ".png");

                count++;
                // write the new file to disk
                try
                {
                    // FileStream is IDisposable
                    using (FileStream fs = new FileStream(path, FileMode.Create))
                    {
                        encoder.Save(fs);
                    }

                    this.StatusText = string.Format(Properties.Resources.SavedScreenshotStatusTextFormat, path);
                }
                catch (IOException)
                {
                    this.StatusText = string.Format(Properties.Resources.FailedScreenshotStatusTextFormat, path);
                }
            }
        }

        /// <summary>
        /// Handles the color frame data arriving from the sensor
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void Reader_ColorFrameArrived(object sender, ColorFrameArrivedEventArgs e)
        {
            // ColorFrame is IDisposable
            using (ColorFrame colorFrame = e.FrameReference.AcquireFrame())
            {
                if (colorFrame != null)
                {
                    FrameDescription colorFrameDescription = colorFrame.FrameDescription;

                    using (KinectBuffer colorBuffer = colorFrame.LockRawImageBuffer())
                    {
                        this.colorBitmap.Lock();

                        // verify data and write the new color frame data to the display bitmap
                        if ((colorFrameDescription.Width == this.colorBitmap.PixelWidth) && (colorFrameDescription.Height == this.colorBitmap.PixelHeight))
                        {
                            colorFrame.CopyConvertedFrameDataToIntPtr(
                                this.colorBitmap.BackBuffer,
                                (uint)(colorFrameDescription.Width * colorFrameDescription.Height * 4),
                                ColorImageFormat.Bgra);

                            this.colorBitmap.AddDirtyRect(new Int32Rect(0, 0, this.colorBitmap.PixelWidth, this.colorBitmap.PixelHeight));
                        }

                        this.colorBitmap.Unlock();
                    }
                }

                // my modification : save current frame as png file. 
                if (this.colorBitmap != null)
                {
                    // create a png bitmap encoder which knows how to save a .png file
                    BitmapEncoder encoder = new PngBitmapEncoder();

                    // create frame from the writable bitmap and add to encoder
                    encoder.Frames.Add(BitmapFrame.Create(this.colorBitmap));

                    string time = count.ToString();
                    //string time = System.DateTime.Now.ToString("hh'-'mm'-'ss", CultureInfo.CurrentUICulture.DateTimeFormat);

                    string myPhotos = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);

                    string path = Path.Combine(myPhotos, "KinectScreenshot-Color-" + time + ".png");

                    count++;
                    // write the new file to disk
                    try
                    {
                        // FileStream is IDisposable
                        using (FileStream fs = new FileStream(path, FileMode.Create))
                        {
                            encoder.Save(fs);
                        }

                        this.StatusText = string.Format(Properties.Resources.SavedScreenshotStatusTextFormat, path);
                    }
                    catch (IOException)
                    {
                        this.StatusText = string.Format(Properties.Resources.FailedScreenshotStatusTextFormat, path);
                    }
                }
            }
        }

        /// <summary>
        /// Handles the event which the sensor becomes unavailable (E.g. paused, closed, unplugged).
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void Sensor_IsAvailableChanged(object sender, IsAvailableChangedEventArgs e)
        {
            // on failure, set the status text
            this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
                                                            : Properties.Resources.SensorNotAvailableStatusText;
        }
    }
}

这段代码的问题是它在录制期间生成了 200 个 png 文件，但是从第 90 帧到最后，所有帧都是相同的（我一运行它就开始录制当我关闭它时停止）。

1) 你能帮我理解为什么会这样吗？为什么它不记录其余的帧并一次又一次地重复一个帧？

2) 关于如何使用 Kinect for Windows v2 以有效的方式和良好的帧速率（比如 20-30 fps）同时将深度和颜色流记录为图像文件，您有什么建议或建议吗？

Answer 1

在不丢失帧的情况下以相同的 fps 同时编写颜色和深度流有点棘手。为了检查相同帧的问题，我建议您尝试将颜色和深度图像写入单独的缓冲区（您也可以将其帧时间戳保存在缓冲区中）并在录制结束后将它们写入磁盘。我会给你一个例子，你可以根据你的问题调整它。

将颜色帧保存到缓冲区：

private void myKinectSensor_ColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
    {
        using (ColorImageFrame color = e.OpenColorImageFrame())
        {
            if (color != null)
            {
                colorbits = new byte[color.PixelDataLength];
                color.CopyPixelDataTo(colorbits);
                image1.Source = BitmapSource.Create(color.Width, color.Height, 96, 96, PixelFormats.Bgr32, null, colorbits, color.Width * color.BytesPerPixel);
                if (StartSavingFrames)
                {
                    SaveColorTimestamps.AddLast(DateTime.Now.ToString("hhmmssfff"));
                    SaveColorFrames.AddLast(colorbits);
                }
            }
        }
    }

将深度帧保存到另一个缓冲区：

private void myKinectSensor_DepthFrameReady(object sender, DepthImageFrameReadyEventArgs e)
    {
        using (DepthImageFrame depth = e.OpenDepthImageFrame())
        {
            depthPixels = new DepthImagePixel[myKinectSensor.DepthStream.FramePixelDataLength];
            if (depth != null)
            {
                frame = new short[depth.PixelDataLength];
                depth.CopyPixelDataTo(frame);
                for (int i = 0; i < frame.Length; i++)
                {
                    frame[i] = (short)(((ushort)frame[i]) >> 3);
                }
                image3.Source = BitmapSource.Create(depth.Width, depth.Height, 96, 96, PixelFormats.Gray16, null, frame, depth.Width * depth.BytesPerPixel);
                if (StartSavingFrames)
                {
                    SaveDepthTimestamps.AddLast(DateTime.Now.ToString("hhmmssfff"));
                    SaveDepthFrames.AddLast(frame);
                }
            }
        }
    }

最后，在每个缓冲区中循环并将帧写入磁盘：

            e = SaveColorTimestamps.GetEnumerator();
            foreach (byte[] node in SaveColorFrames)
            {
                e.MoveNext();
                PngBitmapEncoder enc = new PngBitmapEncoder();
                enc.Frames.Add(BitmapFrame.Create(BitmapSource.Create(640, 480, 96, 96, PixelFormats.Bgr32, null, node, 640*4)));
                string temppath = System.IO.Path.Combine(@"../output/kinect1/color/", e.Current + ".png");
                using (FileStream fs = new FileStream(temppath, FileMode.Create))
                {
                    enc.Save(fs);
                    fs.Close();
                }
            }
            SaveColorTimestamps.Clear();
            SaveColorFrames.Clear();

            e.Dispose();
            e = SaveDepthTimestamps.GetEnumerator();
            foreach (short[] node in SaveDepthFrames)
            {
                e.MoveNext();
                PngBitmapEncoder enc = new PngBitmapEncoder();
                enc.Frames.Add(BitmapFrame.Create(BitmapSource.Create(640, 480, 96, 96, PixelFormats.Gray16, null, node, 640 * 2)));
                string temppath = System.IO.Path.Combine(@"../output/kinect1/depth/", e.Current + ".png");
                using (FileStream fs = new FileStream(temppath, FileMode.Create))
                {
                    enc.Save(fs);
                    fs.Close();
                }
            }

这不是最佳方式，但它会帮助您了解框架编写的工作原理，并且您不会有任何丢帧。