有没有办法在 Unity 中运行神经网络或 GAN？

Question

我只是有一个想法，也许这可能是可能的，但我不知道如何。 像 GPT 或 Stable Diffusion 之类的神经网络可以通过 C# 脚本在 Unity 内部运行吗？如果是这样，怎么做？

Answer 1

是的，首先您需要将模型转换为ONNX。然后，您需要在您的项目中安装Barracuda。下面的代码（我几年前在互联网上找到的）接收一个 ONNX 模型并将一个精灵作为输入提供给它。 （我通常将它用于 CNN 模型）

    [SerializeField] private NNModel modelSource; // ONNX model (asset)
    private Model model; // Runtime model wrapper (binary)
    private IWorker worker; // Barracuda worker for inference
  
    private void Start()
    {
        model = ModelLoader.Load(modelSource); // Load ONNX model as runtime binary model 
        worker = WorkerFactory.CreateWorker(WorkerFactory.Type.ComputePrecompiled, model); // Create Worker
        
    }

    public void Classify(Sprite userInput)
    {
        // the model receives one image, with a fixed width and height and 3 channels (RGB)
        var inputTensor = new Tensor(1, InputHeight, InputWidth, 3);

        var userInputTexture = Resize(userInput.texture, InputWidth, InputHeight);

        var input = userInputTexture.GetPixels32();
        // normalize input
        for (int z = 0, y = 0; y < InputHeight; y++)
        {
            for (var x = 0; x < InputWidth; x++, z++)
            {
                inputTensor[0, InputHeight - y - 1, x, 0] = input[z].b - mean[0];
                inputTensor[0, InputHeight - y - 1, x, 1] = input[z].g - mean[1];
                inputTensor[0, InputHeight - y - 1, x, 2] = input[z].r - mean[2];
            }
        }

        var outTensor = ExecuteInParts(inputTensor);
        var maxVal = Mathf.Max(outTensor.ToReadOnlyArray());
    }

    private Tensor ExecuteInParts(Tensor I, int syncEveryNthLayer = 5)
    {
        var executor = worker.StartManualSchedule(I);
        var it = 0;
        bool hasMoreWork;

        do
        {
            hasMoreWork = executor.MoveNext();
            if (++it % syncEveryNthLayer == 0)
                worker.FlushSchedule();
        } while (hasMoreWork);

        return worker.PeekOutput();
    }

    private static Texture2D Resize(Texture texture2D, int targetX, int targetY)
    {
        var rt = new RenderTexture(targetX, targetY, 24);
        RenderTexture.active = rt;
        Graphics.Blit(texture2D, rt);
        var result = new Texture2D(targetX, targetY);
        result.ReadPixels(new Rect(0, 0, targetX, targetY), 0, 0);
        result.Apply();
        return result;
    }

    private void OnApplicationQuit()
    {
        worker.Dispose();
    }