长篇大论：
很快：
如何处理这些原始 UDP 数据，以便被 H264/MPEG-4 解码器过滤器解码？ 任何人都可以清楚地确定我与 H264/MPEG 流相关的步骤吗？
额外信息：
我可以用 FFmpeg 做到这一点...但我无法真正弄清楚 FFmpeg 如何处理原始数据以便解码器可以解码。
【问题讨论】：
标签： directshow h.264 rtsp rtp mpeg-4
我最近播放了 h264 并遇到了类似的问题。这是我的拆包器课程。我写了一篇很长的博文来节省其他时间来理解这个过程http://cagneymoreau.com/stream-video-android/
Package networking;
import org.apache.commons.logging.Log;
import utility.Debug;
import java.io.Console;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.util.*;
/**
* This class is used to re-assemble udp packets filled with rtp packets into network abstraction layer units
*
*/
public class VideoDecoder {
private static final String TAG = "VideoDecoder";
private PipedOutputStream pipedOutputStream; //this is where we pass the nalus we extract
private Map<Integer, NaluBuffer> assemblyLine = new HashMap<>(); // This holds nalus we are building. Ideally only 1 and if it exceeds 3 there might be a problem
private final int thresh = 30;
private int assemblyThresh = thresh;
private final int trashDelay = 3000;
//unpacking
private final static int HEADER_SIZE = 12;
private final static int rtpByteHeader1 = 128; //rtp header byte 1 should always equal
private final static int typeSPSPPS = 24;
private final static byte typeFUA = 0b01111100;
private final static byte[] startcode = new byte[] { 0x00, 0x00, 0x00, 0x01};
//experimental bools that can mix piped data
private boolean annexB = true; //remove lengths and dd aprefix
private boolean mixed = false; //keep lengths and add pefix dont use with annexb
private boolean prelStyle = false; //include avcc 6 byte data
private boolean directPipe = false; //send in the data with no editing
public VideoDecoder(PipedOutputStream pipedOutputStream)
{
this.pipedOutputStream = pipedOutputStream;
}
// raw udp rtp packets come in here from the the udp.packet.getdata filled at socket
public void addPacket(byte[] incoming)
{
if (directPipe){
transferTOFFmpeg(incoming);
return;
}
if (incoming[0] != (byte) rtpByteHeader1){
System.out.println(TAG + " rtpHeaderError " + Byte.toString(incoming[0]));
}
if (incoming[1] == typeSPSPPS){
System.out.println(TAG + "addPacket type: 24" );
unpackType24(incoming);
}
else if (incoming[1] == typeFUA){
//System.out.println(TAG + "addPacket type: 28" );
unpackType28(incoming);
}
else if (incoming[1] == 1){
System.out.println(TAG + "addPacket type: 1" );
unpackType1(incoming);
}else if (incoming[1] == 5){
System.out.println(TAG + "addPacket type: 5" );
unpackType5(incoming);
}else{
System.out.println(TAG + "addPacket unknown type - ERROR " + String.valueOf(incoming[1]) );
}
}
//SPS & PPS this will get hit before every type 5
//im not rtp compliant.
// length sps length pps prel = 6length
// LL SPSPSPSPSP LL PPSPPSPPSPPS 123456
private void unpackType24(byte[] twentyFour)
{
if (annexB){
int sp = (twentyFour[13] << 8 | twentyFour[14] & 0XFF);
int pp = (twentyFour[sp + 15] << 8 | twentyFour[sp + 16] & 0XFF);
byte[] sps = new byte[sp];
byte[] pps = new byte[pp];
System.arraycopy(twentyFour,15, sps,0,sp);
System.arraycopy(twentyFour,sp + 17, pps,0,pps.length);
transferTOFFmpeg(sps);
transferTOFFmpeg(pps);
}else if (prelStyle)
{
//Debug.debugHex("unpack24 " , twentyFour, twentyFour.length);
int spsl = (twentyFour[14] & 0xff) + 2;
int ppsl = (twentyFour[14+ spsl] & 0xff) +2;
int prel = 6;
byte[] buf = new byte[spsl + ppsl + prel]; //rtp header length - type + experimental data
System.arraycopy(twentyFour, 13, buf, 6,spsl + ppsl);
System.arraycopy(twentyFour, spsl + ppsl + 13, buf,0, 6);
transferTOFFmpeg(buf);
}else{
int spsl = (twentyFour[14] & 0xff) + 2;
int ppsl = (twentyFour[14+ spsl] & 0xff) +2;
byte[] buf = new byte[spsl + ppsl ]; //rtp header length - type + experimental data
System.arraycopy(twentyFour, 13, buf, 0,spsl + ppsl);
//System.arraycopy(twentyFour, spsl + ppsl + 13, buf,0, 6);
transferTOFFmpeg(buf);
}
}
//Single NON IDR Nal - This seems liekly to never occur
private void unpackType1(byte[] one)
{
byte[] buf = new byte[one.length-12];
System.arraycopy(one, 12, buf, 0,buf.length);
transferTOFFmpeg(buf);
}
//Single IDR Nal - This seems likely to never occur
private void unpackType5(byte[] five)
{
byte[] buf = new byte[five.length-12];
System.arraycopy(five, 12, buf, 0,buf.length);
transferTOFFmpeg(buf);
}
// Unpack either any split up nalu - This will get 99.999999 of nalus
synchronized private void unpackType28(byte[] twentyEight)
{
//Debug.deBugHexTrailing("unpack 28 ", twentyEight, 20 );
int ts = (twentyEight[4] << 24 | twentyEight[5] << 16 | twentyEight[6] << 8 | twentyEight[7] & 0XFF); //each nalu has a unique timestamp
//int seqN = (twentyEight[2] << 8 | twentyEight[3] & 0xFF); //each part of that nalu is numbered in order.
// numbers are from every packet ever. not this nalu. no zero or 1 start
//check if already building this nalu
if (assemblyLine.containsKey(ts)){
assemblyLine.get(ts).addPiece(twentyEight);
}
//add a new nalu
else
{
assemblyLine.put(ts, new NaluBuffer(ts, twentyEight));
}
}
//this will transfer the assembled nal units to the media codec/trans-coder/decoder/whatever?!?
private void transferTOFFmpeg(byte[] nalu)
{
Debug.debugHex("VideoDecoder transferTOFFmpg -> ", nalu, 30);
try{
if (annexB || mixed){
pipedOutputStream.write(startcode);
}
pipedOutputStream.write(nalu,0,nalu.length);
}catch (IOException ioe){
System.out.println(TAG + " transferTOFFmpeg - unable to lay pipe ;)");
}
if (assemblyLine.size() > assemblyThresh){
System.err.println(TAG + "transferToFFmpeg -> assemblyLine grows to a count of " + String.valueOf(assemblyLine.size()));
assemblyThresh += thresh;
}
}
private void clearList()
{
String n = "\n";
List<Integer> toremove = new ArrayList<>();
StringBuilder description = new StringBuilder();
for(Map.Entry<Integer, NaluBuffer> entry : assemblyLine.entrySet()) {
Integer key = entry.getKey();
NaluBuffer value = entry.getValue();
if (value.age < System.currentTimeMillis() - trashDelay){
toremove.add(key);
description
.append(String.valueOf(value.timeStamp)).append(" timestamp").append(n)
.append(String.valueOf(value.payloadType)).append(" type").append(n)
.append(String.valueOf(value.count)).append(" count").append(n)
.append(String.valueOf(value.start)).append(" ").append(String.valueOf(value.finish)).append(n)
.append(n);
}
}
for (Integer i :
toremove) {
assemblyLine.remove(i);
}
if (toremove.size() > 0){
System.out.println(TAG + " cleaList current size : " + String.valueOf(assemblyLine.size()) + n + "deleting: " + toremove.size() + n + description);
assemblyThresh = thresh;
}
}
private void deletMe(int key)
{
assemblyLine.remove(key);
if (assemblyLine.size() > 3){
clearList();
}
}
/*
Once a multipart FU-A rtp packet is found it is added to a hashset containing this class
Here we do everything needed to either complete assembly and send or destroy if not completed due to presumable packet loss
** Example Packet From First FU-A with SER = 100 **
description-> |-------RTP--HEADER------| |FU-A--HEADER| |-NAL--HEADER|
byte index-> 0|1|2|3|4|5|6|7|8|9|10|11| 12|13 14|15|16|17|18
| | | | | | | | |S S R C| | |__header | | | | |__type
| | | | |TIMESTM| |__indicator | | | |__length
| | | |__sequence number | | |__length
| | |____sequence number | |___length
| |__payload |__length
|___version padding extension
*/
private class NaluBuffer
{
private final static String TAG = "NaluBuffer";
//private static final int BUFF_SIZE = 200005; // this is the max nalu size + 5 byte header we searched for in our androids nalu search
long age;
//List<String> sizes = new ArrayList<>();
NaluePiece[] buffer = new NaluePiece[167];
int count = 0;
int start;
int finish;
int timeStamp; //from rtp packets.
int completedSize; //this is number of nalu
int payloadType; //nalu type 5 or 1
int byteLength;
int naluByteArrayLength = 0;
//if it doesnt exist
NaluBuffer(int timeStamp, byte[] piece)
{
//System.out.println(TAG + " constructor " + String.valueOf(timeStamp) );
this.timeStamp = timeStamp;
age = System.currentTimeMillis();
addPieceToBuffer(piece);
count++;
}
//adding another piece
synchronized public void addPiece(byte[] piece)
{
//System.out.println(TAG + " addPiece " + String.valueOf(timeStamp));
addPieceToBuffer(piece);
count++;
}
//add to buffer. incoming data is still raw rtp packet
private void addPieceToBuffer(byte[] piece)
{
//System.out.println(TAG + " addPiecetobuffer " + String.valueOf(piece[13]));
int seqN = (piece[2] << 8 | piece[3] & 0xFF);
//add to buffer
buffer[count] = new NaluePiece(seqN, Arrays.copyOfRange(piece, 14,piece.length)); // 14 because we skip rtp header of 12 and fu-a header of 2
int in = ( piece.length - 14); //we save each byte[] copied size so we can easily construct a completed array later
//sizes.add(String.valueOf(in));
naluByteArrayLength += in;
//check if first or last, completed size type etc
if ((start == 0) && (piece[13] & 0b11000000) == 0b10000000){
//start of nalu
start = (piece[2] << 8 | piece[3] & 0xFF);
//type
payloadType = (piece[13] & 0b00011111); //could have used [18] //get type
byteLength = (piece[17]&0xFF | (piece[16]&0xFF)<<8 | (piece[15]&0xFF)<<16 | (piece[14]&0xFF)<<24); //get the h264 encoded length
byteLength += 4; //Now add 4 bytes for the length encoding itself
if (payloadType == 1 || payloadType == 5 && byteLength < 200000){
}else{
System.err.println(TAG + " addpiecetobuffer type: " + String.valueOf(payloadType) + "length: " + String.valueOf(byteLength) );
}
//System.out.println(TAG + " addpiecetobuffer start " + String.valueOf(start) + " type " + String.valueOf(payloadType));
}else if ((finish == 0) && (piece[13] & 0b11000000) == 0b01000000){
//end of nalu
finish = (piece[2] << 8 | piece[3] & 0xFF);
//System.out.println(TAG + " addpiecetobuffer finish " + String.valueOf(finish));
}
if (finish != 0 && start != 0 && completedSize == 0){
//completed size in packet sequnce number NOT in byte length
completedSize = finish - start;
//System.out.println(TAG + " addpiecetobuffer completedsize " + String.valueOf(completedSize));
//originally put in bytes but thats not what I was counting ...duh!
// (piece[14] <<24 | piece[15] << 16 | piece[16] << 8 | piece[17] & 0xFF);
}
//check if complete
if (completedSize != 0 && count == completedSize){
assembleDeliver();
}
}
// we have every sequence number accounted for.
// reconstruct the nalu and send it to the decoder
private void assembleDeliver()
{
count++; //make up for the ount that didn't get called following addpiecetobuffer method
// System.out.println(TAG + " assembleDeliver " + String.valueOf(timeStamp));
//create a new array the exact length needed and sort each nalu by sequence number
NaluePiece[] newbuf = new NaluePiece[count];
System.arraycopy(buffer,0,newbuf,0, count);
Arrays.sort(newbuf);
// TODO: 9/28/2018 we have no gaps in data here checking newbuff !!!!!
//this will be an array we feed/pipe to our videoprocessor
byte[] out;
if (annexB){
out = new byte[naluByteArrayLength-4]; //remove the 4 bytes of length
int tally = 0;
int destPos = 0;
int src = 4;
for (int i = 0; i < count; i++) {
if (i == 1){
src = 0;
}
tally += newbuf[i].piece.length;
System.arraycopy(newbuf[i].piece, src, out, destPos, newbuf[i].piece.length - src);
//Debug.fillCompleteNalData(out, destPos, newbuf[i].piece.length);
destPos += newbuf[i].piece.length - src;
}
/*
StringBuilder sb = new StringBuilder();
sb.append("VideoDecoder assembleDeliver out.length ").append(String.valueOf(out.length))
.append(" destPos ").append(String.valueOf(destPos)).append(" tally ").append(String.valueOf(tally))
.append(" count ").append(String.valueOf(count)).append(" obuf ").append(String.valueOf(completedSize));
for (String s :
sizes) {
sb.append(s).append(" ");
}
System.out.println(sb.toString());
*/
}else{
out = new byte[naluByteArrayLength];
int destPos = 0;
for (int i = 0; i < count; i++) {
System.arraycopy(newbuf[i].piece, 0, out, destPos, newbuf[i].piece.length);
destPos += newbuf[i].piece.length;
}
}
if (naluByteArrayLength != byteLength){
System.err.println(TAG + " assembleDeliver -> ERROR - h264 encoded length: " + String.valueOf(byteLength) + " and byte length found: " + String.valueOf(naluByteArrayLength) + " do not match");
}
// TODO: 9/28/2018 we have gaps in data here
//Debug.checkNaluData(out);
transferTOFFmpeg(out);
deletMe(timeStamp);
}
}
//This class stores the payload and ordering info
private class NaluePiece implements Comparable<NaluePiece>
{
int sequenceNumber; //here is the number we can access to order them
byte[] piece; //here we store the raw payload data to be aggregated
public NaluePiece(int sequenceNumber, byte[] piece)
{
this.sequenceNumber = sequenceNumber;
this.piece = piece;
//Debug.checkNaluPieceData(piece);
}
@Override
public int compareTo(NaluePiece o) {
return Integer.compare(this.sequenceNumber, o.sequenceNumber);
}
}
}
【讨论】：
平安无事！
1.获取数据
如我所见，您已经知道如何做到这一点（启动 RTSP 会话，设置 RTP/AVP/UDP;unicast; 传输，并获取用户数据报）...但如果您有疑问，请询问。
无论是传输方式（UDP还是TCP），数据格式主要是相同的：
[RTP Header - 12bytes][Video data]
[RTP Data]
[$ - 1byte][Transport Channel - 1byte][RTP data length - 2bytes][RTP
data]
因此，要从 UDP 获取数据，您只需剥离代表 RTP 标头的前 12 个字节。但请注意，您需要它来获取视频时序信息，而对于 MPEG4，您需要它来获取打包信息！
对于 TCP，您需要读取第一个字节，直到获得字节 $。然后读取下一个字节，这将是以下数据所属的传输通道（当服务器响应 SETUP 请求时，它说：Transport: RTP/AVP/TCP;unicast;interleaved=0-1 这意味着 VIDEO DATA 将具有 TRANSPORT_CHANNEL=0 并且 VIDEO RTCP DATA 将具有 TRANSPORT_CHANNEL=1）。你想获得 VIDEO DATA，所以我们期望 0... 然后读取一个短的（2 字节），它代表后面的 RTP 数据的长度，所以读取那么多字节，现在做与 UDP 相同的操作。
2。解包数据
H264 和 MPEG4 数据通常是打包的（在 SDP 中有 packetization-mode 参数可以有值 0、1 和 2 分别代表什么意思，以及如何解包，您可以查看 HERE），因为有一个端点可以通过称为 MTU 的 TCP 或 UDP 发送的特定网络限制。它通常为 1500 字节或更少。因此，如果视频帧大于（通常是），则需要将其分段（打包）为 MTU 大小的片段。这可以通过 TCP 和 UDP 传输上的编码器/流媒体来完成，或者您可以在 IP 上中继以在另一侧对视频帧进行分段和重新组合...如果您希望通过 UDP 获得平滑的容易出错的视频，第一个会更好和 TCP。
H264：要检查 RTP 数据（通过 UDP 到达，或通过 TCP 交错）是否包含一个较大的 H264 视频帧的片段，您必须知道该片段在打包时的样子：
H264 片段
First byte: [ 3 NAL UNIT BITS | 5 FRAGMENT TYPE BITS]
Second byte: [ START BIT | END BIT | RESERVED BIT | 5 NAL UNIT BITS]
Other bytes: [... VIDEO FRAGMENT DATA...]
现在，获取名为Data 的字节数组中的第一个 VIDEO DATA 并获取以下信息：
int fragment_type = Data[0] & 0x1F;
int nal_type = Data[1] & 0x1F;
int start_bit = Data[1] & 0x80;
int end_bit = Data[1] & 0x40;
如果fragment_type == 28，那么它后面的视频数据代表视频帧片段。下一个检查是start_bit 设置，如果是，则该片段是序列中的第一个。您可以使用它通过从第一个有效负载字节 (3 NAL UNIT BITS) 获取前 3 位并将它们与来自第二个有效负载字节 (5 NAL UNIT BITS) 的最后 5 位组合来重建 IDR 的 NAL 字节，这样您就会得到一个像这样的字节[3 NAL UNIT BITS | 5 NAL UNIT BITS] .然后首先将该 NAL 字节写入带有来自该片段的VIDEO FRAGMENT DATA 的清除缓冲区。
如果start_bit 和end_bit 为0，则只需将VIDEO FRAGMENT DATA（跳过标识片段的前两个有效负载字节）写入缓冲区。
如果start_bit 为 0 而end_bit 为 1，这意味着它是最后一个片段，您只需将其VIDEO FRAGMENT DATA（跳过标识片段的前两个字节）写入缓冲区，现在你已经重建了你的视频帧！
请记住，RTP 数据在前 12 个字节中包含 RTP 标头，并且如果帧被分段，您永远不会在碎片整理缓冲区中写入前两个字节，并且您需要重建 NAL 字节并首先写入。如果你在这里搞砸了，图片将是局部的（一半是灰色或黑色，否则你会看到伪影）。
MPEG4：
这是个简单的。您需要检查 RTP Header 中的 MARKER_BIT。如果视频数据代表整个视频帧，则设置该字节（1），并且视频数据的0是一个视频帧片段。所以要解包，你需要看看 MARKER_BIT 是什么。如果是1 就是这样，只需读取视频数据字节。
全帧：
[MARKER = 1]
打包帧：
[MARKER = 0], [MARKER = 0], [MARKER = 0], [MARKER = 1]
具有MARKER_BIT=0 的第一个数据包是第一个视频帧片段，随后的所有其他数据包，包括第一个带有MARKER_BIT=1 的数据包都是同一视频帧的片段。所以你需要做的是：
MARKER_BIT=0 将视频数据放入解包缓冲区MARKER_BIT=1 处的下一个视频数据放入同一缓冲区中3.解码器处理数据（NAL 字节流）
解包视频帧后，需要制作 NAL 字节流。格式如下：
0x000001[SPS], 0x000001[PPS], 0x000001[VIDEO FRAME], 0x000001...
0x000001[Visual Object Sequence Start], 0x000001[VIDEO FRAME]
规则：
0x000001 3 字节代码sprop-parameter-sets），对于 MPEG4，VOS 帧（SDP 中的config 参数）因此，您需要为 H264 和 MPEG4 构建一个配置缓冲区，前面带有 3 个字节 0x000001，首先发送它，然后在每个解包后的视频帧前面添加相同的 3 个字节，然后将其发送到解码器。
如果您需要任何澄清，请发表评论... :)
【讨论】：
我有这个@https://net7mma.codeplex.com/的实现
这里是相关代码
/// <summary>
/// Implements Packetization and Depacketization of packets defined in <see href="https://tools.ietf.org/html/rfc6184">RFC6184</see>.
/// </summary>
public class RFC6184Frame : Rtp.RtpFrame
{
/// <summary>
/// Emulation Prevention
/// </summary>
static byte[] NalStart = { 0x00, 0x00, 0x01 };
public RFC6184Frame(byte payloadType) : base(payloadType) { }
public RFC6184Frame(Rtp.RtpFrame existing) : base(existing) { }
public RFC6184Frame(RFC6184Frame f) : this((Rtp.RtpFrame)f) { Buffer = f.Buffer; }
public System.IO.MemoryStream Buffer { get; set; }
/// <summary>
/// Creates any <see cref="Rtp.RtpPacket"/>'s required for the given nal
/// </summary>
/// <param name="nal">The nal</param>
/// <param name="mtu">The mtu</param>
public virtual void Packetize(byte[] nal, int mtu = 1500)
{
if (nal == null) return;
int nalLength = nal.Length;
int offset = 0;
if (nalLength >= mtu)
{
//Make a Fragment Indicator with start bit
byte[] FUI = new byte[] { (byte)(1 << 7), 0x00 };
bool marker = false;
while (offset < nalLength)
{
//Set the end bit if no more data remains
if (offset + mtu > nalLength)
{
FUI[0] |= (byte)(1 << 6);
marker = true;
}
else if (offset > 0) //For packets other than the start
{
//No Start, No End
FUI[0] = 0;
}
//Add the packet
Add(new Rtp.RtpPacket(2, false, false, marker, PayloadTypeByte, 0, SynchronizationSourceIdentifier, HighestSequenceNumber + 1, 0, FUI.Concat(nal.Skip(offset).Take(mtu)).ToArray()));
//Move the offset
offset += mtu;
}
} //Should check for first byte to be 1 - 23?
else Add(new Rtp.RtpPacket(2, false, false, true, PayloadTypeByte, 0, SynchronizationSourceIdentifier, HighestSequenceNumber + 1, 0, nal));
}
/// <summary>
/// Creates <see cref="Buffer"/> with a H.264 RBSP from the contained packets
/// </summary>
public virtual void Depacketize() { bool sps, pps, sei, slice, idr; Depacketize(out sps, out pps, out sei, out slice, out idr); }
/// <summary>
/// Parses all contained packets and writes any contained Nal Units in the RBSP to <see cref="Buffer"/>.
/// </summary>
/// <param name="containsSps">Indicates if a Sequence Parameter Set was found</param>
/// <param name="containsPps">Indicates if a Picture Parameter Set was found</param>
/// <param name="containsSei">Indicates if Supplementatal Encoder Information was found</param>
/// <param name="containsSlice">Indicates if a Slice was found</param>
/// <param name="isIdr">Indicates if a IDR Slice was found</param>
public virtual void Depacketize(out bool containsSps, out bool containsPps, out bool containsSei, out bool containsSlice, out bool isIdr)
{
containsSps = containsPps = containsSei = containsSlice = isIdr = false;
DisposeBuffer();
this.Buffer = new MemoryStream();
//Get all packets in the frame
foreach (Rtp.RtpPacket packet in m_Packets.Values.Distinct())
ProcessPacket(packet, out containsSps, out containsPps, out containsSei, out containsSlice, out isIdr);
//Order by DON?
this.Buffer.Position = 0;
}
/// <summary>
/// Depacketizes a single packet.
/// </summary>
/// <param name="packet"></param>
/// <param name="containsSps"></param>
/// <param name="containsPps"></param>
/// <param name="containsSei"></param>
/// <param name="containsSlice"></param>
/// <param name="isIdr"></param>
internal protected virtual void ProcessPacket(Rtp.RtpPacket packet, out bool containsSps, out bool containsPps, out bool containsSei, out bool containsSlice, out bool isIdr)
{
containsSps = containsPps = containsSei = containsSlice = isIdr = false;
//Starting at offset 0
int offset = 0;
//Obtain the data of the packet (without source list or padding)
byte[] packetData = packet.Coefficients.ToArray();
//Cache the length
int count = packetData.Length;
//Must have at least 2 bytes
if (count <= 2) return;
//Determine if the forbidden bit is set and the type of nal from the first byte
byte firstByte = packetData[offset];
//bool forbiddenZeroBit = ((firstByte & 0x80) >> 7) != 0;
byte nalUnitType = (byte)(firstByte & Common.Binary.FiveBitMaxValue);
//o The F bit MUST be cleared if all F bits of the aggregated NAL units are zero; otherwise, it MUST be set.
//if (forbiddenZeroBit && nalUnitType <= 23 && nalUnitType > 29) throw new InvalidOperationException("Forbidden Zero Bit is Set.");
//Determine what to do
switch (nalUnitType)
{
//Reserved - Ignore
case 0:
case 30:
case 31:
{
return;
}
case 24: //STAP - A
case 25: //STAP - B
case 26: //MTAP - 16
case 27: //MTAP - 24
{
//Move to Nal Data
++offset;
//Todo Determine if need to Order by DON first.
//EAT DON for ALL BUT STAP - A
if (nalUnitType != 24) offset += 2;
//Consume the rest of the data from the packet
while (offset < count)
{
//Determine the nal unit size which does not include the nal header
int tmp_nal_size = Common.Binary.Read16(packetData, offset, BitConverter.IsLittleEndian);
offset += 2;
//If the nal had data then write it
if (tmp_nal_size > 0)
{
//For DOND and TSOFFSET
switch (nalUnitType)
{
case 25:// MTAP - 16
{
//SKIP DOND and TSOFFSET
offset += 3;
goto default;
}
case 26:// MTAP - 24
{
//SKIP DOND and TSOFFSET
offset += 4;
goto default;
}
default:
{
//Read the nal header but don't move the offset
byte nalHeader = (byte)(packetData[offset] & Common.Binary.FiveBitMaxValue);
if (nalHeader > 5)
{
if (nalHeader == 6)
{
Buffer.WriteByte(0);
containsSei = true;
}
else if (nalHeader == 7)
{
Buffer.WriteByte(0);
containsPps = true;
}
else if (nalHeader == 8)
{
Buffer.WriteByte(0);
containsSps = true;
}
}
if (nalHeader == 1) containsSlice = true;
if (nalHeader == 5) isIdr = true;
//Done reading
break;
}
}
//Write the start code
Buffer.Write(NalStart, 0, 3);
//Write the nal header and data
Buffer.Write(packetData, offset, tmp_nal_size);
//Move the offset past the nal
offset += tmp_nal_size;
}
}
return;
}
case 28: //FU - A
case 29: //FU - B
{
/*
Informative note: When an FU-A occurs in interleaved mode, it
always follows an FU-B, which sets its DON.
* Informative note: If a transmitter wants to encapsulate a single
NAL unit per packet and transmit packets out of their decoding
order, STAP-B packet type can be used.
*/
//Need 2 bytes
if (count > 2)
{
//Read the Header
byte FUHeader = packetData[++offset];
bool Start = ((FUHeader & 0x80) >> 7) > 0;
//bool End = ((FUHeader & 0x40) >> 6) > 0;
//bool Receiver = (FUHeader & 0x20) != 0;
//if (Receiver) throw new InvalidOperationException("Receiver Bit Set");
//Move to data
++offset;
//Todo Determine if need to Order by DON first.
//DON Present in FU - B
if (nalUnitType == 29) offset += 2;
//Determine the fragment size
int fragment_size = count - offset;
//If the size was valid
if (fragment_size > 0)
{
//If the start bit was set
if (Start)
{
//Reconstruct the nal header
//Use the first 3 bits of the first byte and last 5 bites of the FU Header
byte nalHeader = (byte)((firstByte & 0xE0) | (FUHeader & Common.Binary.FiveBitMaxValue));
//Could have been SPS / PPS / SEI
if (nalHeader > 5)
{
if (nalHeader == 6)
{
Buffer.WriteByte(0);
containsSei = true;
}
else if (nalHeader == 7)
{
Buffer.WriteByte(0);
containsPps = true;
}
else if (nalHeader == 8)
{
Buffer.WriteByte(0);
containsSps = true;
}
}
if (nalHeader == 1) containsSlice = true;
if (nalHeader == 5) isIdr = true;
//Write the start code
Buffer.Write(NalStart, 0, 3);
//Write the re-construced header
Buffer.WriteByte(nalHeader);
}
//Write the data of the fragment.
Buffer.Write(packetData, offset, fragment_size);
}
}
return;
}
default:
{
// 6 SEI, 7 and 8 are SPS and PPS
if (nalUnitType > 5)
{
if (nalUnitType == 6)
{
Buffer.WriteByte(0);
containsSei = true;
}
else if (nalUnitType == 7)
{
Buffer.WriteByte(0);
containsPps = true;
}
else if (nalUnitType == 8)
{
Buffer.WriteByte(0);
containsSps = true;
}
}
if (nalUnitType == 1) containsSlice = true;
if (nalUnitType == 5) isIdr = true;
//Write the start code
Buffer.Write(NalStart, 0, 3);
//Write the nal heaer and data data
Buffer.Write(packetData, offset, count - offset);
return;
}
}
}
internal void DisposeBuffer()
{
if (Buffer != null)
{
Buffer.Dispose();
Buffer = null;
}
}
public override void Dispose()
{
if (Disposed) return;
base.Dispose();
DisposeBuffer();
}
//To go to an Image...
//Look for a SliceHeader in the Buffer
//Decode Macroblocks in Slice
//Convert Yuv to Rgb
}
还有各种其他 RFC 的实现，它们有助于让媒体在 MediaElement 或其他软件中播放，或者只是将其保存到磁盘。
正在写入容器格式。
【讨论】：
使用 UDP 数据包，您会收到 H.264 流的位，您应该将其解包到 H.264 NAL Units，而反过来，您通常会从过滤器推入 DirectShow 管道。
NAL 单元将被格式化为 DirectShow 媒体样本，也可能作为媒体类型的一部分 (SPS/PPS NAL 单元)。
解包步骤在RFC 6184 - RTP Payload Format for H.264 Video 中描述。这是 RTP 流量的有效负载部分，由RFC 3550 - RTP: A Transport Protocol for Real-Time Applications 定义。
清晰，但不是很短。
【讨论】：