MediaCodec - 如何同步混合来自两个解码器 MediaCodec 对象的音频样本？

Question

我当前的项目需要我将视频的音轨与音频文件混合，我已经设法使用以下代码做到了：

while (mCopyAudio && !audioInternalDecoderDone && pendingInternalAudioDecoderOutputBufferIndex == -1 && (encoderOutputAudioFormat == null || muxing)) {
    int decoderOutputBufferIndex = this.internalAudioDecoder.dequeueOutputBuffer(audioInternalDecoderOutputBufferInfo, TIMEOUT_USEC);
    if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
        break;
    }
    if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
        audioInternalDecoderOutputBuffers = this.internalAudioDecoder.getOutputBuffers();
        break;
    }
    if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
        decoderOutputAudioFormat = this.internalAudioDecoder.getOutputFormat();
        iDecoderOutputChannelNum = decoderOutputAudioFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
        iDecoderOutputAudioSampleRate = decoderOutputAudioFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
        break;
    }
    if ((audioInternalDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {           
        //Not in indent because I couldn't fit it in the editor
      this.internalAudioDecoder.releaseOutputBuffer(decoderOutputBufferIndex,
            false);
        break;
    }
    pendingInternalAudioDecoderOutputBufferIndex = decoderOutputBufferIndex;
    audioDecodedFrameCount++;
    break;
}
while (mCopyAudio && !audioExternalDecoderDone && pendingExternalAudioDecoderOutputBufferIndex == -1 && (encoderOutputAudioFormat == null || muxing)) {
    int decoderOutputBufferIndex = this.externalAudioDecoder.dequeueOutputBuffer(audioExternalDecoderOutputBufferInfo, TIMEOUT_USEC);
    if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
        break;
    }
    if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
        audioExternalDecoderOutputBuffers = this.externalAudioDecoder.getOutputBuffers();
        break;
    }
    if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
        decoderOutputAudioFormat = this.externalAudioDecoder.getOutputFormat();
        eDecoderOutputChannelNum = decoderOutputAudioFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
        eDecoderOutputAudioSampleRate = decoderOutputAudioFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
        break;
    }
    if ((audioExternalDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
        //Not in indent because I couldn't fit it in the editor
     this.externalAudioDecoder.releaseOutputBuffer(decoderOutputBufferIndex,
                false);
        break;
    }
    pendingExternalAudioDecoderOutputBufferIndex = decoderOutputBufferIndex;
    audioDecodedFrameCount++;
    break;
}

while (mCopyAudio && pendingInternalAudioDecoderOutputBufferIndex != -1 && pendingExternalAudioDecoderOutputBufferIndex != -1) {
    int encoderInputBufferIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_USEC);
    if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
        break;
    }
    ByteBuffer encoderInputBuffer = audioEncoderInputBuffers[encoderInputBufferIndex];
    int size = audioInternalDecoderOutputBufferInfo.size;
    long presentationTime = audioInternalDecoderOutputBufferInfo.presentationTimeUs - musicStartUs;

    if (size >= 0) {
        ByteBuffer iDecoderOutputBuffer = audioInternalDecoderOutputBuffers[pendingInternalAudioDecoderOutputBufferIndex].duplicate();
        ByteBuffer eDecoderOutputBuffer = audioExternalDecoderOutputBuffers[pendingExternalAudioDecoderOutputBufferIndex].duplicate();
        byte[] initContents = new byte[ audioInternalDecoderOutputBufferInfo.size];
        byte[] eInitContents = new byte[audioExternalDecoderOutputBufferInfo.size];
        iDecoderOutputBuffer.get(initContents, 0, audioInternalDecoderOutputBufferInfo.size);
        eDecoderOutputBuffer.get(eInitContents, 0, audioExternalDecoderOutputBufferInfo.size);

        /*
        The following is my attempt at compensating for different buffer sizes and timestamps - when the internal and external decoder buffer infos' timestampUs don't sync up with each other. This hasn't gone well.

        if(audioExternalDecoderOutputBufferInfo.presentationTimeUs <= totalTime) {
            if (eInitContents.length > initContents.length) {
                SliceAndRemainder sar = sliceArray(eInitContents, initContents.length - remainderForNextBB.length);
                Log.i("slice_and_remainder", sar.slice.length+" "+sar.remainder.length);
                if(remainderForNextBB.length == initContents.length) {
                    eInitContents = remainderForNextBB;
                    remainderForNextBB = new byte[]{};
                } else {
                    eInitContents = concatTwoArrays(remainderForNextBB, sar.slice);
                    remainderForNextBB = sar.remainder;
                }
            }else if(eInitContents.length < initContents.length) {
                eInitContents = minorUpsamplingFrom44kTo48k(eInitContents);
            }
        }

        For brevity's sake, this code is commented out, so assume the ideal condition that the timestamps in both decoders are synced up properly
        */

        byte[] alteredIContents = scaleByteArrayByScalar(initContents, internalAudioGain);
        byte[] alteredEContents = scaleByteArrayByScalar(eInitContents, externalAudioGain);
        ByteBuffer endByteBuffer;
        if(audioExternalDecoderOutputBufferInfo.presentationTimeUs <= totalTime) {
            byte[] res = mixTwoByteArrays(alteredIContents, alteredEContents, alteredEContents.length);
            Log.i("bytebuffer_mixed_len", res.length+"");
            endByteBuffer = ByteBuffer.wrap(res);
        } else {
            endByteBuffer = ByteBuffer.wrap(alteredIContents);
        }
        iDecoderOutputBuffer.position(audioInternalDecoderOutputBufferInfo.offset);
        iDecoderOutputBuffer.limit(audioInternalDecoderOutputBufferInfo.offset + size);
        encoderInputBuffer.position(0);
        encoderInputBuffer.put(endByteBuffer);
        if((presentationTime < totalTime)) {
            Log.i("presentation_time", presentationTime+" "+totalTime);
            audioEncoder.queueInputBuffer(encoderInputBufferIndex, 0, size, presentationTime, audioInternalDecoderOutputBufferInfo.flags);
        } else {
            audioEncoder.queueInputBuffer(encoderInputBufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
        }
    }
    this.internalAudioDecoder.releaseOutputBuffer(pendingInternalAudioDecoderOutputBufferIndex, false);
    this.externalAudioDecoder.releaseOutputBuffer(pendingExternalAudioDecoderOutputBufferIndex, false);
    pendingInternalAudioDecoderOutputBufferIndex = -1;
    pendingExternalAudioDecoderOutputBufferIndex = -1;
    if ((audioInternalDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
        lastAudioDecoderFinalFrameTimestamp += temporaryAudioDecoderTimestamp + 33333;
        temporaryAudioDecoderTimestamp = 0;
        audioDecoderTimestampOffset = lastAudioDecoderFinalFrameTimestamp;
        audioInternalDecoderDone = true;
        audioExternalDecoderDone = true;
    }
    break;
}

基本上，创建两个提取器-解码器对并将它们汇集到第三个while() 块中进行混合和处理，其中mixTwoByteArrays() 是：

private byte[] mixTwoByteArrays(byte[] src, byte[] with, int numOfMixSamples) {
    final int length = Math.min(src.length, numOfMixSamples);
    byte[] result = new byte[length];
    for(int i = 0; i < length; i++) {
        result[i]=(byte)Math.min(0.999f,((float)src[i]+(float)with[i]));
    }
    return result;
}

如上面注释掉的代码中所述，这适用于时间戳彼此同步的音轨/文件。我的问题是在他们不这样做的情况下——比如最近，音轨的时间戳是 26666 的倍数，而音频文件的时间戳是 27000 的倍数。

我曾考虑单独处理音轨，然后将结果与原始视频轨道合并，但这会对处理时间产生不利影响，所以我宁愿实时处理并使用它作为最后的解决方案。

有没有办法实时处理？

Answer 1

就像准备一个ArrayList<Byte>() 来将外部解码器样本中的所有字节放入其中一样简单。然后从 ArrayList 中取出前 4096 个（或任何内部解码器的缓冲区信息的 size 是）字节（然后从所述 ArrayList 的索引 0 中删除所述样本数）与内部解码器的样本混合.

private ArrayList<Byte> externalBytesArrayList = new ArrayList<Byte>();

//All the other stuff omitted

while (mCopyAudio && pendingInternalAudioDecoderOutputBufferIndex != -1 && pendingExternalAudioDecoderOutputBufferIndex != -1) {
    int encoderInputBufferIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_USEC);

    ByteBuffer encoderInputBuffer = audioEncoderInputBuffers[encoderInputBufferIndex];
    int size = audioInternalDecoderOutputBufferInfo.size;
    long presentationTime = audioInternalDecoderOutputBufferInfo.presentationTimeUs - musicStartUs;
    if (size >= 0) {

        ByteBuffer iDecoderOutputBuffer = audioInternalDecoderOutputBuffers[pendingInternalAudioDecoderOutputBufferIndex].duplicate();
        ByteBuffer eDecoderOutputBuffer = audioExternalDecoderOutputBuffers[pendingExternalAudioDecoderOutputBufferIndex].duplicate();

        byte[] initContents = new byte[ audioInternalDecoderOutputBufferInfo.size];
        byte[] eInitContents = new byte[audioExternalDecoderOutputBufferInfo.size];

        iDecoderOutputBuffer.get(initContents, 0, audioInternalDecoderOutputBufferInfo.size);
        eDecoderOutputBuffer.get(eInitContents, 0, audioExternalDecoderOutputBufferInfo.size);
        externalBytesArrayList.addAll(Bytes.asList(eInitContents));

        byte[] eContents;
        //Here: take the first 4096 bytes from the external decoder's sample, save the rest in the ArrayList
        //I need to replace 4096 with audioInternalDecoderOutputBufferInfo.presentationTimeUs - though complications might follow.
        if(!(4096 > externalBytesArrayList.size())) {
            List<Byte> subset = externalBytesArrayList.subList(0, 4096);
            eContents = Bytes.toArray(subset);
            externalBytesArrayList.subList(0, 4096).clear();
        }else {
            eContents = new byte[audioInternalDecoderOutputBufferInfo.size];
        }

        byte[] alteredIContents = scaleByteArrayByScalar(initContents, internalAudioGain);
        byte[] alteredEContents = scaleByteArrayByScalar(eContents, externalAudioGain);
        ByteBuffer endByteBuffer;
        byte[] res = mixTwoByteArrays(alteredIContents, alteredEContents, alteredEContents.length);
        endByteBuffer = ByteBuffer.wrap(res);

        iDecoderOutputBuffer.position(audioInternalDecoderOutputBufferInfo.offset);
        iDecoderOutputBuffer.limit(audioInternalDecoderOutputBufferInfo.offset + size);

        encoderInputBuffer.position(0);
        encoderInputBuffer.put(endByteBuffer);

        if((presentationTime < totalTime)) {
            Log.i("presentation_time", presentationTime+" "+totalTime);
            audioEncoder.queueInputBuffer(encoderInputBufferIndex, 0, size, presentationTime, audioInternalDecoderOutputBufferInfo.flags);
        } else {
            audioEncoder.queueInputBuffer(encoderInputBufferIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
        }
    }
    this.internalAudioDecoder.releaseOutputBuffer(pendingInternalAudioDecoderOutputBufferIndex, false);
    this.externalAudioDecoder.releaseOutputBuffer(pendingExternalAudioDecoderOutputBufferIndex, false);
    pendingInternalAudioDecoderOutputBufferIndex = -1;
    pendingExternalAudioDecoderOutputBufferIndex = -1;

    if ((audioInternalDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
        lastAudioDecoderFinalFrameTimestamp += temporaryAudioDecoderTimestamp + 33333;
        temporaryAudioDecoderTimestamp = 0;
        audioDecoderTimestampOffset = lastAudioDecoderFinalFrameTimestamp;
        audioInternalDecoderDone = true;
        audioExternalDecoderDone = true;
    }
    break;
}