lywsvip/ZLMediaKit
1
0
Fork 0
mirror of https://github.com/ZLMediaKit/ZLMediaKit.git synced 2026-05-06 23:33:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add support of codec plugin

Browse Source
This commit is contained in:
xia-chu
2023-12-10 10:21:40 +08:00
committed by 夏楚
parent 9ff4c09211
commit 25c99470ee
42 changed files with 772 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

433
ext-codec/AAC.cpp Normal file
View File

@@ -0,0 +1,433 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "AAC.h"
#include "AACRtp.h"
#include "AACRtmp.h"
#include "Common/Parser.h"
#include "Extension/Factory.h"
#ifdef ENABLE_MP4
#include "mpeg4-aac.h"
#endif
using namespace std;
using namespace toolkit;
namespace mediakit{
#ifndef ENABLE_MP4
unsigned const samplingFrequencyTable[16] = { 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350, 0, 0, 0 };
class AdtsHeader {
public:
unsigned int syncword = 0; // 12 bslbf 同步字The bit string ‘1111 1111 1111’，说明一个ADTS帧的开始
unsigned int id; // 1 bslbf MPEG 标示符, 设置为1
unsigned int layer; // 2 uimsbf Indicates which layer is used. Set to ‘00’
unsigned int protection_absent; // 1 bslbf 表示是否误码校验
unsigned int profile; // 2 uimsbf 表示使用哪个级别的AAC，如01 Low Complexity(LC)--- AACLC
unsigned int sf_index; // 4 uimsbf 表示使用的采样率下标
unsigned int private_bit; // 1 bslbf
unsigned int channel_configuration; // 3 uimsbf 表示声道数
unsigned int original; // 1 bslbf
unsigned int home; // 1 bslbf
// 下面的为改变的参数即每一帧都不同
unsigned int copyright_identification_bit; // 1 bslbf
unsigned int copyright_identification_start; // 1 bslbf
unsigned int aac_frame_length; // 13 bslbf 一个ADTS帧的长度包括ADTS头和raw data block
unsigned int adts_buffer_fullness; // 11 bslbf 0x7FF 说明是码率可变的码流
// no_raw_data_blocks_in_frame 表示ADTS帧中有number_of_raw_data_blocks_in_frame + 1个AAC原始帧.
// 所以说number_of_raw_data_blocks_in_frame == 0
// 表示说ADTS帧中有一个AAC数据块并不是说没有。(一个AAC原始帧包含一段时间内1024个采样及相关数据)
unsigned int no_raw_data_blocks_in_frame; // 2 uimsfb
};
static void dumpAdtsHeader(const AdtsHeader &hed, uint8_t *out) {
out[0] = (hed.syncword >> 4 & 0xFF); // 8bit
out[1] = (hed.syncword << 4 & 0xF0); // 4 bit
out[1] |= (hed.id << 3 & 0x08); // 1 bit
out[1] |= (hed.layer << 1 & 0x06); // 2bit
out[1] |= (hed.protection_absent & 0x01); // 1 bit
out[2] = (hed.profile << 6 & 0xC0); // 2 bit
out[2] |= (hed.sf_index << 2 & 0x3C); // 4bit
out[2] |= (hed.private_bit << 1 & 0x02); // 1 bit
out[2] |= (hed.channel_configuration >> 2 & 0x03); // 1 bit
out[3] = (hed.channel_configuration << 6 & 0xC0); // 2 bit
out[3] |= (hed.original << 5 & 0x20); // 1 bit
out[3] |= (hed.home << 4 & 0x10); // 1 bit
out[3] |= (hed.copyright_identification_bit << 3 & 0x08); // 1 bit
out[3] |= (hed.copyright_identification_start << 2 & 0x04); // 1 bit
out[3] |= (hed.aac_frame_length >> 11 & 0x03); // 2 bit
out[4] = (hed.aac_frame_length >> 3 & 0xFF); // 8 bit
out[5] = (hed.aac_frame_length << 5 & 0xE0); // 3 bit
out[5] |= (hed.adts_buffer_fullness >> 6 & 0x1F); // 5 bit
out[6] = (hed.adts_buffer_fullness << 2 & 0xFC); // 6 bit
out[6] |= (hed.no_raw_data_blocks_in_frame & 0x03); // 2 bit
}
static bool parseAacConfig(const string &config, AdtsHeader &adts) {
if (config.size() < 2) {
return false;
}
uint8_t cfg1 = config[0];
uint8_t cfg2 = config[1];
int audioObjectType;
int sampling_frequency_index;
int channel_configuration;
audioObjectType = cfg1 >> 3;
sampling_frequency_index = ((cfg1 & 0x07) << 1) | (cfg2 >> 7);
channel_configuration = (cfg2 & 0x7F) >> 3;
adts.syncword = 0x0FFF;
adts.id = 0;
adts.layer = 0;
adts.protection_absent = 1;
adts.profile = audioObjectType - 1;
adts.sf_index = sampling_frequency_index;
adts.private_bit = 0;
adts.channel_configuration = channel_configuration;
adts.original = 0;
adts.home = 0;
adts.copyright_identification_bit = 0;
adts.copyright_identification_start = 0;
adts.aac_frame_length = 7;
adts.adts_buffer_fullness = 2047;
adts.no_raw_data_blocks_in_frame = 0;
return true;
}
#endif// ENABLE_MP4
int getAacFrameLength(const uint8_t *data, size_t bytes) {
uint16_t len;
if (bytes < 7) return -1;
if (0xFF != data[0] || 0xF0 != (data[1] & 0xF0)) {
return -1;
}
len = ((uint16_t) (data[3] & 0x03) << 11) | ((uint16_t) data[4] << 3) | ((uint16_t) (data[5] >> 5) & 0x07);
return len;
}
string makeAacConfig(const uint8_t *hex, size_t length){
#ifndef ENABLE_MP4
if (!(hex[0] == 0xFF && (hex[1] & 0xF0) == 0xF0)) {
return "";
}
// Get and check the 'profile':
unsigned char profile = (hex[2] & 0xC0) >> 6; // 2 bits
if (profile == 3) {
return "";
}
// Get and check the 'sampling_frequency_index':
unsigned char sampling_frequency_index = (hex[2] & 0x3C) >> 2; // 4 bits
if (samplingFrequencyTable[sampling_frequency_index] == 0) {
return "";
}
// Get and check the 'channel_configuration':
unsigned char channel_configuration = ((hex[2] & 0x01) << 2) | ((hex[3] & 0xC0) >> 6); // 3 bits
unsigned char audioSpecificConfig[2];
unsigned char const audioObjectType = profile + 1;
audioSpecificConfig[0] = (audioObjectType << 3) | (sampling_frequency_index >> 1);
audioSpecificConfig[1] = (sampling_frequency_index << 7) | (channel_configuration << 3);
return string((char *)audioSpecificConfig,2);
#else
struct mpeg4_aac_t aac;
memset(&aac, 0, sizeof(aac));
if (mpeg4_aac_adts_load(hex, length, &aac) > 0) {
char buf[32] = {0};
int len = mpeg4_aac_audio_specific_config_save(&aac, (uint8_t *) buf, sizeof(buf));
if (len > 0) {
return string(buf, len);
}
}
WarnL << "生成aac config失败, adts header:" << hexdump(hex, length);
return "";
#endif
}
int dumpAacConfig(const string &config, size_t length, uint8_t *out, size_t out_size) {
#ifndef ENABLE_MP4
AdtsHeader header;
parseAacConfig(config, header);
header.aac_frame_length = (decltype(header.aac_frame_length))(ADTS_HEADER_LEN + length);
dumpAdtsHeader(header, out);
return ADTS_HEADER_LEN;
#else
struct mpeg4_aac_t aac;
memset(&aac, 0, sizeof(aac));
int ret = mpeg4_aac_audio_specific_config_load((uint8_t *) config.data(), config.size(), &aac);
if (ret > 0) {
ret = mpeg4_aac_adts_save(&aac, length, out, out_size);
}
if (ret < 0) {
WarnL << "生成adts头失败:" << ret << ", aac config:" << hexdump(config.data(), config.size());
}
assert((int)out_size >= ret);
return ret;
#endif
}
bool parseAacConfig(const string &config, int &samplerate, int &channels) {
#ifndef ENABLE_MP4
AdtsHeader header;
if (!parseAacConfig(config, header)) {
return false;
}
samplerate = samplingFrequencyTable[header.sf_index];
channels = header.channel_configuration;
return true;
#else
struct mpeg4_aac_t aac;
memset(&aac, 0, sizeof(aac));
int ret = mpeg4_aac_audio_specific_config_load((uint8_t *) config.data(), config.size(), &aac);
if (ret > 0) {
samplerate = aac.sampling_frequency;
channels = aac.channels;
return true;
}
WarnL << "获取aac采样率、声道数失败:" << hexdump(config.data(), config.size());
return false;
#endif
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/**
* aac类型SDP
*/
class AACSdp : public Sdp {
public:
/**
* 构造函数
* @param aac_cfg aac两个字节的配置描述
* @param payload_type rtp payload type
* @param sample_rate 音频采样率
* @param channels 通道数
* @param bitrate 比特率
*/
AACSdp(const string &aac_cfg, int payload_type, int sample_rate, int channels, int bitrate)
: Sdp(sample_rate, payload_type) {
_printer << "m=audio 0 RTP/AVP " << payload_type << "\r\n";
if (bitrate) {
_printer << "b=AS:" << bitrate << "\r\n";
}
_printer << "a=rtpmap:" << payload_type << " " << getCodecName(CodecAAC) << "/" << sample_rate << "/" << channels << "\r\n";
string configStr;
char buf[4] = { 0 };
for (auto &ch : aac_cfg) {
snprintf(buf, sizeof(buf), "%02X", (uint8_t)ch);
configStr.append(buf);
}
_printer << "a=fmtp:" << payload_type << " streamtype=5;profile-level-id=1;mode=AAC-hbr;"
<< "sizelength=13;indexlength=3;indexdeltalength=3;config=" << configStr << "\r\n";
}
string getSdp() const override { return _printer; }
private:
_StrPrinter _printer;
};
////////////////////////////////////////////////////////////////////////////////////////////////////
AACTrack::AACTrack(const string &aac_cfg) {
if (aac_cfg.size() < 2) {
throw std::invalid_argument("adts配置必须最少2个字节");
}
_cfg = aac_cfg;
update();
}
CodecId AACTrack::getCodecId() const {
return CodecAAC;
}
bool AACTrack::ready() const {
return !_cfg.empty();
}
int AACTrack::getAudioSampleRate() const {
return _sampleRate;
}
int AACTrack::getAudioSampleBit() const {
return _sampleBit;
}
int AACTrack::getAudioChannel() const {
return _channel;
}
static Frame::Ptr addADTSHeader(const Frame::Ptr &frame_in, const std::string &aac_config) {
auto frame = FrameImp::create();
frame->_codec_id = CodecAAC;
// 生成adts头
char adts_header[32] = { 0 };
auto size = dumpAacConfig(aac_config, frame_in->size(), (uint8_t *)adts_header, sizeof(adts_header));
CHECK(size > 0, "Invalid adts config");
frame->_prefix_size = size;
frame->_dts = frame_in->dts();
frame->_buffer.assign(adts_header, size);
frame->_buffer.append(frame_in->data(), frame_in->size());
frame->setIndex(frame_in->getIndex());
return frame;
}
bool AACTrack::inputFrame(const Frame::Ptr &frame) {
if (!frame->prefixSize()) {
CHECK(ready());
return inputFrame_l(addADTSHeader(frame, _cfg));
}
bool ret = false;
//有adts头，尝试分帧
int64_t dts = frame->dts();
int64_t pts = frame->pts();
auto ptr = frame->data();
auto end = frame->data() + frame->size();
while (ptr < end) {
auto frame_len = getAacFrameLength((uint8_t *)ptr, end - ptr);
if (frame_len < ADTS_HEADER_LEN) {
break;
}
if (frame_len == (int)frame->size()) {
return inputFrame_l(frame);
}
auto sub_frame = std::make_shared<FrameInternalBase<FrameFromPtr>>(frame, (char *)ptr, frame_len, dts, pts, ADTS_HEADER_LEN);
ptr += frame_len;
if (ptr > end) {
WarnL << "invalid aac length in adts header: " << frame_len
<< ", remain data size: " << end - (ptr - frame_len);
break;
}
if (inputFrame_l(sub_frame)) {
ret = true;
}
dts += 1024 * 1000 / getAudioSampleRate();
pts += 1024 * 1000 / getAudioSampleRate();
}
return ret;
}
bool AACTrack::inputFrame_l(const Frame::Ptr &frame) {
if (_cfg.empty() && frame->prefixSize()) {
// 未获取到aac_cfg信息，根据7个字节的adts头生成aac config
_cfg = makeAacConfig((uint8_t *)(frame->data()), frame->prefixSize());
update();
}
if (frame->size() > frame->prefixSize()) {
// 除adts头外，有实际负载
return AudioTrack::inputFrame(frame);
}
return false;
}
toolkit::Buffer::Ptr AACTrack::getExtraData() const {
CHECK(ready());
return std::make_shared<BufferString>(_cfg);
}
void AACTrack::setExtraData(const uint8_t *data, size_t size) {
CHECK(size >= 2);
_cfg.assign((char *)data, size);
update();
}
bool AACTrack::update() {
return parseAacConfig(_cfg, _sampleRate, _channel);
}
Track::Ptr AACTrack::clone() const {
return std::make_shared<AACTrack>(*this);
}
Sdp::Ptr AACTrack::getSdp(uint8_t payload_type) const {
if (!ready()) {
WarnL << getCodecName() << " Track未准备好";
return nullptr;
}
return std::make_shared<AACSdp>(getExtraData()->toString(), payload_type, getAudioSampleRate(), getAudioChannel(), getBitRate() / 1024);
}
namespace {
CodecId getCodec() {
return CodecAAC;
}
Track::Ptr getTrackByCodecId(int sample_rate, int channels, int sample_bit) {
return std::make_shared<AACTrack>();
}
Track::Ptr getTrackBySdp(const SdpTrack::Ptr &track) {
string aac_cfg_str = findSubString(track->_fmtp.data(), "config=", ";");
if (aac_cfg_str.empty()) {
aac_cfg_str = findSubString(track->_fmtp.data(), "config=", nullptr);
}
if (aac_cfg_str.empty()) {
// 如果sdp中获取不到aac config信息，那么在rtp也无法获取，那么忽略该Track
return nullptr;
}
string aac_cfg;
for (size_t i = 0; i < aac_cfg_str.size() / 2; ++i) {
unsigned int cfg;
sscanf(aac_cfg_str.substr(i * 2, 2).data(), "%02X", &cfg);
cfg &= 0x00FF;
aac_cfg.push_back((char)cfg);
}
return std::make_shared<AACTrack>(aac_cfg);
}
RtpCodec::Ptr getRtpEncoderByCodecId(uint8_t pt) {
return std::make_shared<AACRtpEncoder>();
}
RtpCodec::Ptr getRtpDecoderByCodecId() {
return std::make_shared<AACRtpDecoder>();
}
RtmpCodec::Ptr getRtmpEncoderByTrack(const Track::Ptr &track) {
return std::make_shared<AACRtmpEncoder>(track);
}
RtmpCodec::Ptr getRtmpDecoderByTrack(const Track::Ptr &track) {
return std::make_shared<AACRtmpDecoder>(track);
}
size_t aacPrefixSize(const char *data, size_t bytes) {
uint8_t *ptr = (uint8_t *)data;
size_t prefix = 0;
if (!(bytes > ADTS_HEADER_LEN && ptr[0] == 0xFF && (ptr[1] & 0xF0) == 0xF0)) {
return 0;
}
return ADTS_HEADER_LEN;
}
Frame::Ptr getFrameFromPtr(const char *data, size_t bytes, uint64_t dts, uint64_t pts) {
return std::make_shared<FrameFromPtr>(CodecAAC, (char *)data, bytes, dts, pts, aacPrefixSize(data, bytes));
}
} // namespace
CodecPlugin aac_plugin = { getCodec,
getTrackByCodecId,
getTrackBySdp,
getRtpEncoderByCodecId,
getRtpDecoderByCodecId,
getRtmpEncoderByTrack,
getRtmpDecoderByTrack,
getFrameFromPtr };
} // namespace mediakit

57
ext-codec/AAC.h Normal file
View File

@@ -0,0 +1,57 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_AAC_H
#define ZLMEDIAKIT_AAC_H
#include "Extension/Frame.h"
#include "Extension/Track.h"
#define ADTS_HEADER_LEN 7
namespace mediakit{
/**
* aac音频通道
*/
class AACTrack : public AudioTrack {
public:
using Ptr = std::shared_ptr<AACTrack>;
AACTrack() = default;
/**
* 通过aac extra data 构造对象
*/
AACTrack(const std::string &aac_cfg);
bool ready() const override;
CodecId getCodecId() const override;
int getAudioChannel() const override;
int getAudioSampleRate() const override;
int getAudioSampleBit() const override;
bool inputFrame(const Frame::Ptr &frame) override;
toolkit::Buffer::Ptr getExtraData() const override;
void setExtraData(const uint8_t *data, size_t size) override;
bool update() override;
private:
Sdp::Ptr getSdp(uint8_t payload_type) const override;
Track::Ptr clone() const override;
bool inputFrame_l(const Frame::Ptr &frame);
private:
std::string _cfg;
int _channel = 0;
int _sampleRate = 0;
int _sampleBit = 16;
};
}//namespace mediakit
#endif //ZLMEDIAKIT_AAC_H

66
ext-codec/AACRtmp.cpp Normal file
View File

@@ -0,0 +1,66 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "AACRtmp.h"
#include "Rtmp/Rtmp.h"
using namespace std;
using namespace toolkit;
namespace mediakit {
void AACRtmpDecoder::inputRtmp(const RtmpPacket::Ptr &pkt) {
CHECK(pkt->size() > 2);
if (pkt->isConfigFrame()) {
getTrack()->setExtraData((uint8_t *)pkt->data() + 2, pkt->size() - 2);
return;
}
RtmpCodec::inputFrame(std::make_shared<FrameFromPtr>(CodecAAC, pkt->buffer.data() + 2, pkt->buffer.size() - 2, pkt->time_stamp));
}
/////////////////////////////////////////////////////////////////////////////////////
bool AACRtmpEncoder::inputFrame(const Frame::Ptr &frame) {
auto pkt = RtmpPacket::create();
// header
pkt->buffer.push_back(_audio_flv_flags);
pkt->buffer.push_back((uint8_t)RtmpAACPacketType::aac_raw);
// aac data
pkt->buffer.append(frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize());
pkt->body_size = pkt->buffer.size();
pkt->chunk_id = CHUNK_AUDIO;
pkt->stream_index = STREAM_MEDIA;
pkt->time_stamp = frame->dts();
pkt->type_id = MSG_AUDIO;
RtmpCodec::inputRtmp(pkt);
return true;
}
void AACRtmpEncoder::makeConfigPacket() {
_audio_flv_flags = getAudioRtmpFlags(getTrack());
auto pkt = RtmpPacket::create();
// header
pkt->buffer.push_back(_audio_flv_flags);
pkt->buffer.push_back((uint8_t)RtmpAACPacketType::aac_config_header);
// aac config
auto extra_data = getTrack()->getExtraData();
CHECK(extra_data);
pkt->buffer.append(extra_data->data(), extra_data->size());
pkt->body_size = pkt->buffer.size();
pkt->chunk_id = CHUNK_AUDIO;
pkt->stream_index = STREAM_MEDIA;
pkt->time_stamp = 0;
pkt->type_id = MSG_AUDIO;
RtmpCodec::inputRtmp(pkt);
}
}//namespace mediakit

67
ext-codec/AACRtmp.h Normal file
View File

@@ -0,0 +1,67 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_AACRTMPCODEC_H
#define ZLMEDIAKIT_AACRTMPCODEC_H
#include "AAC.h"
#include "Rtmp/RtmpCodec.h"
#include "Extension/Track.h"
namespace mediakit {
/**
* aac Rtmp转adts类
*/
class AACRtmpDecoder : public RtmpCodec {
public:
using Ptr = std::shared_ptr<AACRtmpDecoder>;
AACRtmpDecoder(const Track::Ptr &track) : RtmpCodec(track) {}
/**
* 输入Rtmp并解码
* @param rtmp Rtmp数据包
*/
void inputRtmp(const RtmpPacket::Ptr &rtmp) override;
};
/**
* aac adts转Rtmp类
*/
class AACRtmpEncoder : public RtmpCodec {
public:
using Ptr = std::shared_ptr<AACRtmpEncoder>;
/**
* 构造函数，track可以为空，此时则在inputFrame时输入adts头
* 如果track不为空且包含adts头相关信息，
* 那么inputFrame时可以不输入adts头
* @param track
*/
AACRtmpEncoder(const Track::Ptr &track) : RtmpCodec(track) {}
/**
* 输入aac 数据，可以不带adts头
* @param frame aac数据
*/
bool inputFrame(const Frame::Ptr &frame) override;
/**
* 生成config包
*/
void makeConfigPacket() override;
private:
uint8_t _audio_flv_flags {0};
};
}//namespace mediakit
#endif //ZLMEDIAKIT_AACRTMPCODEC_H

128
ext-codec/AACRtp.cpp Normal file
View File

@@ -0,0 +1,128 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "AACRtp.h"
namespace mediakit{
bool AACRtpEncoder::inputFrame(const Frame::Ptr &frame) {
auto ptr = (char *)frame->data() + frame->prefixSize();
auto size = frame->size() - frame->prefixSize();
auto remain_size = size;
auto max_size = getRtpInfo().getMaxSize() - 4;
while (remain_size > 0) {
if (remain_size <= max_size) {
outputRtp(ptr, remain_size, size, true, frame->dts());
break;
}
outputRtp(ptr, max_size, size, false, frame->dts());
ptr += max_size;
remain_size -= max_size;
}
return true;
}
void AACRtpEncoder::outputRtp(const char *data, size_t len, size_t total_len, bool mark, uint64_t stamp) {
auto rtp = getRtpInfo().makeRtp(TrackAudio, nullptr, len + 4, mark, stamp);
auto payload = rtp->data() + RtpPacket::kRtpTcpHeaderSize + RtpPacket::kRtpHeaderSize;
payload[0] = 0;
payload[1] = 16;
payload[2] = ((total_len) >> 5) & 0xFF;
payload[3] = ((total_len & 0x1F) << 3) & 0xFF;
memcpy(payload + 4, data, len);
RtpCodec::inputRtp(std::move(rtp), false);
}
/////////////////////////////////////////////////////////////////////////////////////
AACRtpDecoder::AACRtpDecoder() {
obtainFrame();
}
void AACRtpDecoder::obtainFrame() {
//从缓存池重新申请对象，防止覆盖已经写入环形缓存的对象
_frame = FrameImp::create();
_frame->_codec_id = CodecAAC;
}
bool AACRtpDecoder::inputRtp(const RtpPacket::Ptr &rtp, bool key_pos) {
auto payload_size = rtp->getPayloadSize();
if (payload_size <= 0) {
//无实际负载
return false;
}
auto stamp = rtp->getStampMS();
//rtp数据开始部分
auto ptr = rtp->getPayload();
//rtp数据末尾
auto end = ptr + payload_size;
//首2字节表示Au-Header的个数，单位bit，所以除以16得到Au-Header个数
auto au_header_count = ((ptr[0] << 8) | ptr[1]) >> 4;
if (!au_header_count) {
//问题issue: https://github.com/ZLMediaKit/ZLMediaKit/issues/1869
WarnL << "invalid aac rtp au_header_count";
return false;
}
//记录au_header起始指针
auto au_header_ptr = ptr + 2;
ptr = au_header_ptr + au_header_count * 2;
if (end < ptr) {
//数据不够
return false;
}
if (!_last_dts) {
//记录第一个时间戳
_last_dts = stamp;
}
//每个audio unit时间戳增量
auto dts_inc = (stamp - _last_dts) / au_header_count;
if (dts_inc < 0 && dts_inc > 100) {
//时间戳增量异常，忽略
dts_inc = 0;
}
for (int i = 0; i < au_header_count; ++i) {
// 之后的2字节是AU_HEADER,其中高13位表示一帧AAC负载的字节长度，低3位无用
uint16_t size = ((au_header_ptr[0] << 8) | au_header_ptr[1]) >> 3;
if (ptr + size > end) {
//数据不够
break;
}
if (size) {
//设置aac数据
_frame->_buffer.assign((char *) ptr, size);
//设置当前audio unit时间戳
_frame->_dts = _last_dts + i * dts_inc;
ptr += size;
au_header_ptr += 2;
flushData();
}
}
//记录上次时间戳
_last_dts = stamp;
return false;
}
void AACRtpDecoder::flushData() {
auto ptr = reinterpret_cast<const uint8_t *>(_frame->data());
if ((ptr[0] == 0xFF && (ptr[1] & 0xF0) == 0xF0) && _frame->size() > ADTS_HEADER_LEN) {
// adts头打入了rtp包，不符合规范，兼容EasyPusher的bug
_frame->_prefix_size = ADTS_HEADER_LEN;
}
RtpCodec::inputFrame(_frame);
obtainFrame();
}
}//namespace mediakit

64
ext-codec/AACRtp.h Normal file
View File

@@ -0,0 +1,64 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_AACRTPCODEC_H
#define ZLMEDIAKIT_AACRTPCODEC_H
#include "AAC.h"
#include "Rtsp/RtpCodec.h"
namespace mediakit {
/**
* aac rtp转adts类
*/
class AACRtpDecoder : public RtpCodec {
public:
using Ptr = std::shared_ptr<AACRtpDecoder>;
AACRtpDecoder();
/**
* 输入rtp并解码
* @param rtp rtp数据包
* @param key_pos 此参数内部强制转换为false,请忽略之
*/
bool inputRtp(const RtpPacket::Ptr &rtp, bool key_pos = false) override;
private:
void obtainFrame();
void flushData();
private:
uint64_t _last_dts = 0;
FrameImp::Ptr _frame;
};
/**
* aac adts转rtp类
*/
class AACRtpEncoder : public RtpCodec {
public:
using Ptr = std::shared_ptr<AACRtpEncoder>;
/**
* 输入aac 数据，必须带dats头
* @param frame 带dats头的aac数据
*/
bool inputFrame(const Frame::Ptr &frame) override;
private:
void outputRtp(const char *data, size_t len, size_t total_len, bool mark, uint64_t stamp);
};
}//namespace mediakit
#endif //ZLMEDIAKIT_AACRTPCODEC_H

48
ext-codec/CMakeLists.txt Normal file
View File

@@ -0,0 +1,48 @@
# MIT License
#
# Copyright (c) 2016-2022 The ZLMediaKit project authors. All Rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
set(COMPILE_DEFINITIONS)
set(INCLUDE_DIRECTORIES)
file(GLOB EXT_SRC_LIST
${CMAKE_CURRENT_SOURCE_DIR}/*.c
${CMAKE_CURRENT_SOURCE_DIR}/*.cpp
${CMAKE_CURRENT_SOURCE_DIR}/*.h
${CMAKE_CURRENT_SOURCE_DIR}/*.hpp)
add_library(ext-codec STATIC ${EXT_SRC_LIST})
add_library(ZLMediaKit::ext-codec ALIAS ext-codec)
target_compile_options(ext-codec PRIVATE ${COMPILE_OPTIONS_DEFAULT})
target_compile_definitions(ext-codec PUBLIC ${COMPILE_DEFINITIONS})
target_link_libraries(ext-codec PRIVATE ZLMediaKit::MediaKit ZLMediaKit::ToolKit ${LINK_LIBRARIES})
target_include_directories(ext-codec
PRIVATE
"$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>"
PUBLIC
"$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>"
${INCLUDE_DIRECTORIES})
update_cached_list(MK_LINK_LIBRARIES ZLMediaKit::ext-codec ${LINK_LIBRARIES})

188
ext-codec/G711.cpp Normal file
View File

@@ -0,0 +1,188 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "G711.h"
#include "G711Rtp.h"
#include "Extension/Factory.h"
#include "Extension/CommonRtp.h"
#include "Extension/CommonRtmp.h"
using namespace std;
using namespace toolkit;
namespace mediakit {
/**
* G711类型SDP
*/
class G711Sdp : public Sdp {
public:
/**
* G711采样率固定为8000
* @param codecId G711A G711U
* @param payload_type rtp payload type
* @param sample_rate 音频采样率
* @param channels 通道数
* @param bitrate 比特率
*/
G711Sdp(CodecId codecId, int payload_type, int sample_rate, int channels, int bitrate)
: Sdp(sample_rate, payload_type) {
_printer << "m=audio 0 RTP/AVP " << payload_type << "\r\n";
if (bitrate) {
_printer << "b=AS:" << bitrate << "\r\n";
}
_printer << "a=rtpmap:" << payload_type << " " << getCodecName(codecId) << "/" << sample_rate << "/" << channels << "\r\n";
}
string getSdp() const override {
return _printer;
}
private:
_StrPrinter _printer;
};
Track::Ptr G711Track::clone() const {
return std::make_shared<G711Track>(*this);
}
Sdp::Ptr G711Track::getSdp(uint8_t payload_type) const {
if (!ready()) {
WarnL << getCodecName() << " Track未准备好";
return nullptr;
}
const auto codec = getCodecId();
const auto sample_rate = getAudioSampleRate();
const auto audio_channel = getAudioChannel();
const auto bitrate = getBitRate() >> 10;
if (sample_rate == 8000 && audio_channel == 1) {
// https://datatracker.ietf.org/doc/html/rfc3551#section-6
payload_type = (codec == CodecG711U) ? Rtsp::PT_PCMU : Rtsp::PT_PCMA;
}
return std::make_shared<G711Sdp>(codec, payload_type, sample_rate, audio_channel, bitrate);
}
namespace {
CodecId getCodecA() {
return CodecG711A;
}
CodecId getCodecU() {
return CodecG711A;
}
Track::Ptr getTrackByCodecId_l(CodecId codec, int sample_rate, int channels, int sample_bit) {
return (sample_rate && channels && sample_bit) ? std::make_shared<G711Track>(codec, sample_rate, channels, sample_bit) : nullptr;
}
Track::Ptr getTrackByCodecIdA(int sample_rate, int channels, int sample_bit) {
return getTrackByCodecId_l(CodecG711A, sample_rate, channels, sample_bit);
}
Track::Ptr getTrackByCodecIdU(int sample_rate, int channels, int sample_bit) {
return getTrackByCodecId_l(CodecG711U, sample_rate, channels, sample_bit);
}
Track::Ptr getTrackBySdp_l(CodecId codec, const SdpTrack::Ptr &track) {
return std::make_shared<G711Track>(codec, track->_samplerate, track->_channel, 16);
}
Track::Ptr getTrackBySdpA(const SdpTrack::Ptr &track) {
return getTrackBySdp_l(CodecG711A, track);
}
Track::Ptr getTrackBySdpU(const SdpTrack::Ptr &track) {
return getTrackBySdp_l(CodecG711U, track);
}
RtpCodec::Ptr getRtpEncoderByCodecId_l(CodecId codec, uint8_t pt) {
if (pt == Rtsp::PT_PCMA || pt == Rtsp::PT_PCMU) {
return std::make_shared<G711RtpEncoder>(codec, 1);
}
return std::make_shared<CommonRtpEncoder>();
}
RtpCodec::Ptr getRtpEncoderByCodecIdA(uint8_t pt) {
return getRtpEncoderByCodecId_l(CodecG711A, pt);
}
RtpCodec::Ptr getRtpEncoderByCodecIdU(uint8_t pt) {
return getRtpEncoderByCodecId_l(CodecG711U, pt);
}
RtpCodec::Ptr getRtpDecoderByCodecId_l(CodecId codec) {
return std::make_shared<CommonRtpDecoder>(codec);
}
RtpCodec::Ptr getRtpDecoderByCodecIdA() {
return getRtpDecoderByCodecId_l(CodecG711A);
}
RtpCodec::Ptr getRtpDecoderByCodecIdU() {
return getRtpDecoderByCodecId_l(CodecG711U);
}
RtmpCodec::Ptr getRtmpEncoderByTrack(const Track::Ptr &track) {
auto audio_track = dynamic_pointer_cast<AudioTrack>(track);
if (audio_track->getAudioSampleRate() != 8000 || audio_track->getAudioChannel() != 1 || audio_track->getAudioSampleBit() != 16) {
//rtmp对g711只支持8000/1/16规格，但是ZLMediaKit可以解析其他规格的G711
WarnL << "RTMP only support G711 with 8000/1/16, now is"
<< audio_track->getAudioSampleRate() << "/"
<< audio_track->getAudioChannel() << "/"
<< audio_track->getAudioSampleBit()
<< ", ignored it";
return nullptr;
}
return std::make_shared<CommonRtmpEncoder>(track);
}
RtmpCodec::Ptr getRtmpDecoderByTrack(const Track::Ptr &track) {
return std::make_shared<CommonRtmpDecoder>(track);
}
Frame::Ptr getFrameFromPtr_l(CodecId codec, const char *data, size_t bytes, uint64_t dts, uint64_t pts) {
return std::make_shared<FrameFromPtr>(codec, (char *)data, bytes, dts, pts);
}
Frame::Ptr getFrameFromPtrA(const char *data, size_t bytes, uint64_t dts, uint64_t pts) {
return getFrameFromPtr_l(CodecG711A, (char *)data, bytes, dts, pts);
}
Frame::Ptr getFrameFromPtrU(const char *data, size_t bytes, uint64_t dts, uint64_t pts) {
return getFrameFromPtr_l(CodecG711U, (char *)data, bytes, dts, pts);
}
} // namespace
CodecPlugin g711a_plugin = { getCodecA,
getTrackByCodecIdA,
getTrackBySdpA,
getRtpEncoderByCodecIdA,
getRtpDecoderByCodecIdA,
getRtmpEncoderByTrack,
getRtmpDecoderByTrack,
getFrameFromPtrA };
CodecPlugin g711u_plugin = { getCodecU,
getTrackByCodecIdU,
getTrackBySdpU,
getRtpEncoderByCodecIdU,
getRtpDecoderByCodecIdU,
getRtmpEncoderByTrack,
getRtmpDecoderByTrack,
getFrameFromPtrA };
}//namespace mediakit

33
ext-codec/G711.h Normal file
View File

@@ -0,0 +1,33 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_G711_H
#define ZLMEDIAKIT_G711_H
#include "Extension/Frame.h"
#include "Extension/Track.h"
namespace mediakit{
/**
* G711音频通道
*/
class G711Track : public AudioTrackImp{
public:
using Ptr = std::shared_ptr<G711Track>;
G711Track(CodecId codecId, int sample_rate, int channels, int sample_bit) : AudioTrackImp(codecId, 8000, 1, 16) {}
private:
Sdp::Ptr getSdp(uint8_t payload_type) const override;
Track::Ptr clone() const override;
};
}//namespace mediakit
#endif //ZLMEDIAKIT_G711_H

48
ext-codec/G711Rtp.cpp Normal file
View File

@@ -0,0 +1,48 @@
#include "G711Rtp.h"
namespace mediakit {
G711RtpEncoder::G711RtpEncoder(CodecId codec, uint32_t channels){
_cache_frame = FrameImp::create();
_cache_frame->_codec_id = codec;
_channels = channels;
}
bool G711RtpEncoder::inputFrame(const Frame::Ptr &frame) {
auto dur = (_cache_frame->size() - _cache_frame->prefixSize()) / (8 * _channels);
auto next_pts = _cache_frame->pts() + dur;
if (next_pts == 0) {
_cache_frame->_pts = frame->pts();
} else {
if ((next_pts + 20) < frame->pts()) { // 有丢包超过20ms
_cache_frame->_pts = frame->pts() - dur;
}
}
_cache_frame->_buffer.append(frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize());
auto stamp = _cache_frame->pts();
auto ptr = _cache_frame->data() + _cache_frame->prefixSize();
auto len = _cache_frame->size() - _cache_frame->prefixSize();
auto remain_size = len;
auto max_size = 160 * _channels; // 20 ms per rtp
int n = 0;
bool mark = false;
while (remain_size >= max_size) {
size_t rtp_size;
if (remain_size >= max_size) {
rtp_size = max_size;
} else {
break;
}
n++;
stamp += 20;
RtpCodec::inputRtp(getRtpInfo().makeRtp(TrackAudio, ptr, rtp_size, mark, stamp), false);
ptr += rtp_size;
remain_size -= rtp_size;
}
_cache_frame->_buffer.erase(0, n * max_size);
_cache_frame->_pts += 20 * n;
return len > 0;
}
} // namespace mediakit

45
ext-codec/G711Rtp.h Normal file
View File

@@ -0,0 +1,45 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_G711RTP_H
#define ZLMEDIAKIT_G711RTP_H
#include "Rtsp/RtpCodec.h"
#include "Extension/Frame.h"
#include "Extension/CommonRtp.h"
namespace mediakit {
/**
* G711 rtp编码类
*/
class G711RtpEncoder : public RtpCodec {
public:
using Ptr = std::shared_ptr<G711RtpEncoder>;
/**
* 构造函数
* @param codec 编码类型
* @param channels 通道数
*/
G711RtpEncoder(CodecId codec, uint32_t channels);
/**
* 输入帧数据并编码成rtp
*/
bool inputFrame(const Frame::Ptr &frame) override;
private:
uint32_t _channels = 1;
FrameImp::Ptr _cache_frame;
};
}//namespace mediakit
#endif //ZLMEDIAKIT_G711RTP_H

409
ext-codec/H264.cpp Normal file
View File

@@ -0,0 +1,409 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "H264.h"
#include "H264Rtmp.h"
#include "H264Rtp.h"
#include "SPSParser.h"
#include "Util/logger.h"
#include "Util/base64.h"
#include "Common/Parser.h"
#include "Common/config.h"
#include "Extension/Factory.h"
#ifdef ENABLE_MP4
#include "mpeg4-avc.h"
#endif
using namespace std;
using namespace toolkit;
namespace mediakit {
static bool getAVCInfo(const char *sps, size_t sps_len, int &iVideoWidth, int &iVideoHeight, float &iVideoFps) {
if (sps_len < 4) {
return false;
}
T_GetBitContext tGetBitBuf;
T_SPS tH264SpsInfo;
memset(&tGetBitBuf, 0, sizeof(tGetBitBuf));
memset(&tH264SpsInfo, 0, sizeof(tH264SpsInfo));
tGetBitBuf.pu8Buf = (uint8_t *)sps + 1;
tGetBitBuf.iBufSize = (int)(sps_len - 1);
if (0 != h264DecSeqParameterSet((void *)&tGetBitBuf, &tH264SpsInfo)) {
return false;
}
h264GetWidthHeight(&tH264SpsInfo, &iVideoWidth, &iVideoHeight);
h264GeFramerate(&tH264SpsInfo, &iVideoFps);
// ErrorL << iVideoWidth << " " << iVideoHeight << " " << iVideoFps;
return true;
}
bool getAVCInfo(const string &strSps, int &iVideoWidth, int &iVideoHeight, float &iVideoFps) {
return getAVCInfo(strSps.data(), strSps.size(), iVideoWidth, iVideoHeight, iVideoFps);
}
static const char *memfind(const char *buf, ssize_t len, const char *subbuf, ssize_t sublen) {
for (auto i = 0; i < len - sublen; ++i) {
if (memcmp(buf + i, subbuf, sublen) == 0) {
return buf + i;
}
}
return NULL;
}
void splitH264(
const char *ptr, size_t len, size_t prefix, const std::function<void(const char *, size_t, size_t)> &cb) {
auto start = ptr + prefix;
auto end = ptr + len;
size_t next_prefix;
while (true) {
auto next_start = memfind(start, end - start, "\x00\x00\x01", 3);
if (next_start) {
//找到下一帧
if (*(next_start - 1) == 0x00) {
//这个是00 00 00 01开头
next_start -= 1;
next_prefix = 4;
} else {
//这个是00 00 01开头
next_prefix = 3;
}
//记得加上本帧prefix长度
cb(start - prefix, next_start - start + prefix, prefix);
//搜索下一帧末尾的起始位置
start = next_start + next_prefix;
//记录下一帧的prefix长度
prefix = next_prefix;
continue;
}
//未找到下一帧,这是最后一帧
cb(start - prefix, end - start + prefix, prefix);
break;
}
}
size_t prefixSize(const char *ptr, size_t len) {
if (len < 4) {
return 0;
}
if (ptr[0] != 0x00 || ptr[1] != 0x00) {
//不是0x00 00开头
return 0;
}
if (ptr[2] == 0x00 && ptr[3] == 0x01) {
//是0x00 00 00 01
return 4;
}
if (ptr[2] == 0x01) {
//是0x00 00 01
return 3;
}
return 0;
}
////////////////////////////////////////////////////////////////////////////////////////////////////
H264Track::H264Track(const string &sps, const string &pps, int sps_prefix_len, int pps_prefix_len) {
_sps = sps.substr(sps_prefix_len);
_pps = pps.substr(pps_prefix_len);
update();
}
CodecId H264Track::getCodecId() const {
return CodecH264;
}
int H264Track::getVideoHeight() const {
return _height;
}
int H264Track::getVideoWidth() const {
return _width;
}
float H264Track::getVideoFps() const {
return _fps;
}
bool H264Track::ready() const {
return !_sps.empty() && !_pps.empty();
}
bool H264Track::inputFrame(const Frame::Ptr &frame) {
using H264FrameInternal = FrameInternal<H264FrameNoCacheAble>;
int type = H264_TYPE(frame->data()[frame->prefixSize()]);
if ((type == H264Frame::NAL_B_P || type == H264Frame::NAL_IDR) && ready()) {
return inputFrame_l(frame);
}
//非I/B/P帧情况下，split一下，防止多个帧粘合在一起
bool ret = false;
splitH264(frame->data(), frame->size(), frame->prefixSize(), [&](const char *ptr, size_t len, size_t prefix) {
H264FrameInternal::Ptr sub_frame = std::make_shared<H264FrameInternal>(frame, (char *)ptr, len, prefix);
if (inputFrame_l(sub_frame)) {
ret = true;
}
});
return ret;
}
toolkit::Buffer::Ptr H264Track::getExtraData() const {
CHECK(ready());
#ifdef ENABLE_MP4
struct mpeg4_avc_t avc;
memset(&avc, 0, sizeof(avc));
string sps_pps = string("\x00\x00\x00\x01", 4) + _sps + string("\x00\x00\x00\x01", 4) + _pps;
h264_annexbtomp4(&avc, sps_pps.data(), (int)sps_pps.size(), NULL, 0, NULL, NULL);
std::string extra_data;
extra_data.resize(1024);
auto extra_data_size = mpeg4_avc_decoder_configuration_record_save(&avc, (uint8_t *)extra_data.data(), extra_data.size());
if (extra_data_size == -1) {
WarnL << "生成H264 extra_data 失败";
return nullptr;
}
extra_data.resize(extra_data_size);
return std::make_shared<BufferString>(std::move(extra_data));
#else
std::string extra_data;
// AVCDecoderConfigurationRecord start
extra_data.push_back(1); // version
extra_data.push_back(_sps[1]); // profile
extra_data.push_back(_sps[2]); // compat
extra_data.push_back(_sps[3]); // level
extra_data.push_back((char)0xff); // 6 bits reserved + 2 bits nal size length - 1 (11)
extra_data.push_back((char)0xe1); // 3 bits reserved + 5 bits number of sps (00001)
// sps
uint16_t size = (uint16_t)_sps.size();
size = htons(size);
extra_data.append((char *)&size, 2);
extra_data.append(_sps);
// pps
extra_data.push_back(1); // version
size = (uint16_t)_pps.size();
size = htons(size);
extra_data.append((char *)&size, 2);
extra_data.append(_pps);
return std::make_shared<BufferString>(std::move(extra_data));
#endif
}
void H264Track::setExtraData(const uint8_t *data, size_t bytes) {
#ifdef ENABLE_MP4
struct mpeg4_avc_t avc;
memset(&avc, 0, sizeof(avc));
if (mpeg4_avc_decoder_configuration_record_load(data, bytes, &avc) > 0) {
std::vector<uint8_t> config(bytes * 2);
int size = mpeg4_avc_to_nalu(&avc, config.data(), bytes * 2);
if (size > 4) {
splitH264((char *)config.data(), size, 4, [&](const char *ptr, size_t len, size_t prefix) {
inputFrame_l(std::make_shared<H264FrameNoCacheAble>((char *)ptr, len, 0, 0, prefix));
});
update();
}
}
#else
CHECK(bytes >= 8); // 6 + 2
size_t offset = 6;
uint16_t sps_size = data[offset] << 8 | data[offset + 1];
auto sps_ptr = data + offset + 2;
offset += (2 + sps_size);
CHECK(bytes >= offset + 2); // + pps_size
_sps.assign((char *)sps_ptr, sps_size);
uint16_t pps_size = data[offset] << 8 | data[offset + 1];
auto pps_ptr = data + offset + 2;
offset += (2 + pps_size);
CHECK(bytes >= offset);
_pps.assign((char *)pps_ptr, pps_size);
update();
#endif
}
bool H264Track::update() {
return getAVCInfo(_sps, _width, _height, _fps);
}
Track::Ptr H264Track::clone() const {
return std::make_shared<H264Track>(*this);
}
bool H264Track::inputFrame_l(const Frame::Ptr &frame) {
int type = H264_TYPE(frame->data()[frame->prefixSize()]);
bool ret = true;
switch (type) {
case H264Frame::NAL_SPS: {
_sps = string(frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize());
_latest_is_config_frame = true;
ret = VideoTrack::inputFrame(frame);
break;
}
case H264Frame::NAL_PPS: {
_pps = string(frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize());
_latest_is_config_frame = true;
ret = VideoTrack::inputFrame(frame);
break;
}
default:
// 避免识别不出关键帧
if (_latest_is_config_frame && !frame->dropAble()) {
if (!frame->keyFrame()) {
const_cast<Frame::Ptr &>(frame) = std::make_shared<FrameCacheAble>(frame, true);
}
}
// 判断是否是I帧, 并且如果是,那判断前面是否插入过config帧, 如果插入过就不插入了
if (frame->keyFrame() && !_latest_is_config_frame) {
insertConfigFrame(frame);
}
if(!frame->dropAble()){
_latest_is_config_frame = false;
}
ret = VideoTrack::inputFrame(frame);
break;
}
if (_width == 0 && ready()) {
update();
}
return ret;
}
void H264Track::insertConfigFrame(const Frame::Ptr &frame) {
if (!_sps.empty()) {
auto spsFrame = FrameImp::create<H264Frame>();
spsFrame->_prefix_size = 4;
spsFrame->_buffer.assign("\x00\x00\x00\x01", 4);
spsFrame->_buffer.append(_sps);
spsFrame->_dts = frame->dts();
spsFrame->setIndex(frame->getIndex());
VideoTrack::inputFrame(spsFrame);
}
if (!_pps.empty()) {
auto ppsFrame = FrameImp::create<H264Frame>();
ppsFrame->_prefix_size = 4;
ppsFrame->_buffer.assign("\x00\x00\x00\x01", 4);
ppsFrame->_buffer.append(_pps);
ppsFrame->_dts = frame->dts();
ppsFrame->setIndex(frame->getIndex());
VideoTrack::inputFrame(ppsFrame);
}
}
class H264Sdp : public Sdp {
public:
H264Sdp(const string &strSPS, const string &strPPS, int payload_type, int bitrate) : Sdp(90000, payload_type) {
_printer << "m=video 0 RTP/AVP " << payload_type << "\r\n";
if (bitrate) {
_printer << "b=AS:" << bitrate << "\r\n";
}
_printer << "a=rtpmap:" << payload_type << " " << getCodecName(CodecH264) << "/" << 90000 << "\r\n";
/**
Single NAI Unit Mode = 0. // Single NAI mode (Only nals from 1-23 are allowed)
Non Interleaved Mode = 1，// Non-interleaved Mode: 1-23，24 (STAP-A)，28 (FU-A) are allowed
Interleaved Mode = 2, // 25 (STAP-B)，26 (MTAP16)，27 (MTAP24)，28 (EU-A)，and 29 (EU-B) are allowed.
**/
GET_CONFIG(bool, h264_stap_a, Rtp::kH264StapA);
_printer << "a=fmtp:" << payload_type << " packetization-mode=" << h264_stap_a << "; profile-level-id=";
uint32_t profile_level_id = 0;
if (strSPS.length() >= 4) { // sanity check
profile_level_id = (uint8_t(strSPS[1]) << 16) |
(uint8_t(strSPS[2]) << 8) |
(uint8_t(strSPS[3])); // profile_idc|constraint_setN_flag|level_idc
}
char profile[8];
snprintf(profile, sizeof(profile), "%06X", profile_level_id);
_printer << profile;
_printer << "; sprop-parameter-sets=";
_printer << encodeBase64(strSPS) << ",";
_printer << encodeBase64(strPPS) << "\r\n";
}
string getSdp() const { return _printer; }
private:
_StrPrinter _printer;
};
Sdp::Ptr H264Track::getSdp(uint8_t payload_type) const {
if (!ready()) {
WarnL << getCodecName() << " Track未准备好";
return nullptr;
}
return std::make_shared<H264Sdp>(_sps, _pps, payload_type, getBitRate() / 1024);
}
namespace {
CodecId getCodec() {
return CodecH264;
}
Track::Ptr getTrackByCodecId(int sample_rate, int channels, int sample_bit) {
return std::make_shared<H264Track>();
}
Track::Ptr getTrackBySdp(const SdpTrack::Ptr &track) {
//a=fmtp:96 packetization-mode=1;profile-level-id=42C01F;sprop-parameter-sets=Z0LAH9oBQBboQAAAAwBAAAAPI8YMqA==,aM48gA==
auto map = Parser::parseArgs(track->_fmtp, ";", "=");
auto sps_pps = map["sprop-parameter-sets"];
string base64_SPS = findSubString(sps_pps.data(), NULL, ",");
string base64_PPS = findSubString(sps_pps.data(), ",", NULL);
auto sps = decodeBase64(base64_SPS);
auto pps = decodeBase64(base64_PPS);
if (sps.empty() || pps.empty()) {
//如果sdp里面没有sps/pps,那么可能在后续的rtp里面恢复出sps/pps
return std::make_shared<H264Track>();
}
return std::make_shared<H264Track>(sps, pps, 0, 0);
}
RtpCodec::Ptr getRtpEncoderByCodecId(uint8_t pt) {
return std::make_shared<H264RtpEncoder>();
}
RtpCodec::Ptr getRtpDecoderByCodecId() {
return std::make_shared<H264RtpDecoder>();
}
RtmpCodec::Ptr getRtmpEncoderByTrack(const Track::Ptr &track) {
return std::make_shared<H264RtmpEncoder>(track);
}
RtmpCodec::Ptr getRtmpDecoderByTrack(const Track::Ptr &track) {
return std::make_shared<H264RtmpDecoder>(track);
}
Frame::Ptr getFrameFromPtr(const char *data, size_t bytes, uint64_t dts, uint64_t pts) {
return std::make_shared<H264FrameNoCacheAble>((char *)data, bytes, dts, pts, prefixSize(data, bytes));
}
} // namespace
CodecPlugin h264_plugin = { getCodec,
getTrackByCodecId,
getTrackBySdp,
getRtpEncoderByCodecId,
getRtpDecoderByCodecId,
getRtmpEncoderByTrack,
getRtmpDecoderByTrack,
getFrameFromPtr };
} // namespace mediakit

135
ext-codec/H264.h Normal file
View File

@@ -0,0 +1,135 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_H264_H
#define ZLMEDIAKIT_H264_H
#include "Extension/Frame.h"
#include "Extension/Track.h"
#define H264_TYPE(v) ((uint8_t)(v) & 0x1F)
namespace mediakit{
void splitH264(const char *ptr, size_t len, size_t prefix, const std::function<void(const char *, size_t, size_t)> &cb);
size_t prefixSize(const char *ptr, size_t len);
template<typename Parent>
class H264FrameHelper : public Parent{
public:
friend class FrameImp;
friend class toolkit::ResourcePool_l<H264FrameHelper>;
using Ptr = std::shared_ptr<H264FrameHelper>;
enum {
NAL_IDR = 5,
NAL_SEI = 6,
NAL_SPS = 7,
NAL_PPS = 8,
NAL_AUD = 9,
NAL_B_P = 1,
};
template<typename ...ARGS>
H264FrameHelper(ARGS &&...args): Parent(std::forward<ARGS>(args)...) {
this->_codec_id = CodecH264;
}
bool keyFrame() const override {
auto nal_ptr = (uint8_t *) this->data() + this->prefixSize();
return H264_TYPE(*nal_ptr) == NAL_IDR && decodeAble();
}
bool configFrame() const override {
auto nal_ptr = (uint8_t *) this->data() + this->prefixSize();
switch (H264_TYPE(*nal_ptr)) {
case NAL_SPS:
case NAL_PPS: return true;
default: return false;
}
}
bool dropAble() const override {
auto nal_ptr = (uint8_t *) this->data() + this->prefixSize();
switch (H264_TYPE(*nal_ptr)) {
case NAL_SEI:
case NAL_AUD: return true;
default: return false;
}
}
bool decodeAble() const override {
auto nal_ptr = (uint8_t *) this->data() + this->prefixSize();
auto type = H264_TYPE(*nal_ptr);
//多slice情况下, first_mb_in_slice 表示其为一帧的开始
return type >= NAL_B_P && type <= NAL_IDR && (nal_ptr[1] & 0x80);
}
};
/**
* 264帧类
*/
using H264Frame = H264FrameHelper<FrameImp>;
/**
* 防止内存拷贝的H264类
* 用户可以通过该类型快速把一个指针无拷贝的包装成Frame类
*/
using H264FrameNoCacheAble = H264FrameHelper<FrameFromPtr>;
/**
* 264视频通道
*/
class H264Track : public VideoTrack {
public:
using Ptr = std::shared_ptr<H264Track>;
/**
* 不指定sps pps构造h264类型的媒体
* 在随后的inputFrame中获取sps pps
*/
H264Track() = default;
/**
* 构造h264类型的媒体
* @param sps sps帧数据
* @param pps pps帧数据
* @param sps_prefix_len 264头长度，可以为3个或4个字节，一般为0x00 00 00 01
* @param pps_prefix_len 264头长度，可以为3个或4个字节，一般为0x00 00 00 01
*/
H264Track(const std::string &sps, const std::string &pps, int sps_prefix_len = 4, int pps_prefix_len = 4);
bool ready() const override;
CodecId getCodecId() const override;
int getVideoHeight() const override;
int getVideoWidth() const override;
float getVideoFps() const override;
bool inputFrame(const Frame::Ptr &frame) override;
toolkit::Buffer::Ptr getExtraData() const override;
void setExtraData(const uint8_t *data, size_t size) override;
bool update() override;
private:
Sdp::Ptr getSdp(uint8_t payload_type) const override;
Track::Ptr clone() const override;
bool inputFrame_l(const Frame::Ptr &frame);
void insertConfigFrame(const Frame::Ptr &frame);
private:
bool _latest_is_config_frame = false;
int _width = 0;
int _height = 0;
float _fps = 0;
std::string _sps;
std::string _pps;
};
}//namespace mediakit
#endif //ZLMEDIAKIT_H264_H

109
ext-codec/H264Rtmp.cpp Normal file
View File

@@ -0,0 +1,109 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "Rtmp/utils.h"
#include "H264Rtmp.h"
using namespace std;
using namespace toolkit;
namespace mediakit {
void H264RtmpDecoder::inputRtmp(const RtmpPacket::Ptr &pkt) {
if (pkt->isConfigFrame()) {
CHECK(pkt->size() > 5);
getTrack()->setExtraData((uint8_t *)pkt->data() + 5, pkt->size() - 5);
return;
}
CHECK(pkt->size() > 9);
uint8_t *cts_ptr = (uint8_t *)(pkt->buffer.data() + 2);
int32_t cts = (((cts_ptr[0] << 16) | (cts_ptr[1] << 8) | (cts_ptr[2])) + 0xff800000) ^ 0xff800000;
auto pts = pkt->time_stamp + cts;
splitFrame((uint8_t *)pkt->data() + 5, pkt->size() - 5, pkt->time_stamp, pts);
}
void H264RtmpDecoder::splitFrame(const uint8_t *data, size_t size, uint32_t dts, uint32_t pts) {
auto end = data + size;
while (data + 4 < end) {
uint32_t frame_len = load_be32(data);
data += 4;
if (data + frame_len > end) {
break;
}
outputFrame((const char *)data, frame_len, dts, pts);
data += frame_len;
}
}
void H264RtmpDecoder::outputFrame(const char *data, size_t len, uint32_t dts, uint32_t pts) {
auto frame = FrameImp::create<H264Frame>();
frame->_prefix_size = 4;
frame->_dts = dts;
frame->_pts = pts;
frame->_buffer.assign("\x00\x00\x00\x01", 4); // 添加264头
frame->_buffer.append(data, len);
RtmpCodec::inputFrame(frame);
}
////////////////////////////////////////////////////////////////////////
void H264RtmpEncoder::flush() {
inputFrame(nullptr);
}
bool H264RtmpEncoder::inputFrame(const Frame::Ptr &frame) {
if (!_rtmp_packet) {
_rtmp_packet = RtmpPacket::create();
//flags/not config/cts预占位
_rtmp_packet->buffer.resize(5);
}
return _merger.inputFrame(frame, [this](uint64_t dts, uint64_t pts, const Buffer::Ptr &, bool have_key_frame) {
// flags
_rtmp_packet->buffer[0] = (uint8_t)RtmpVideoCodec::h264 | ((uint8_t)(have_key_frame ? RtmpFrameType::key_frame : RtmpFrameType::inter_frame) << 4);
_rtmp_packet->buffer[1] = (uint8_t)RtmpH264PacketType::h264_nalu;
int32_t cts = pts - dts;
// cts
set_be24(&_rtmp_packet->buffer[2], cts);
_rtmp_packet->time_stamp = dts;
_rtmp_packet->body_size = _rtmp_packet->buffer.size();
_rtmp_packet->chunk_id = CHUNK_VIDEO;
_rtmp_packet->stream_index = STREAM_MEDIA;
_rtmp_packet->type_id = MSG_VIDEO;
// 输出rtmp packet
RtmpCodec::inputRtmp(_rtmp_packet);
_rtmp_packet = nullptr;
}, &_rtmp_packet->buffer);
}
void H264RtmpEncoder::makeConfigPacket() {
auto flags = (uint8_t)RtmpVideoCodec::h264;
flags |= ((uint8_t)RtmpFrameType::key_frame << 4);
auto pkt = RtmpPacket::create();
// header
pkt->buffer.push_back(flags);
pkt->buffer.push_back((uint8_t)RtmpH264PacketType::h264_config_header);
// cts
pkt->buffer.append("\x0\x0\x0", 3);
// AVCDecoderConfigurationRecord start
auto extra_data = getTrack()->getExtraData();
CHECK(extra_data);
pkt->buffer.append(extra_data->data(), extra_data->size());
pkt->body_size = pkt->buffer.size();
pkt->chunk_id = CHUNK_VIDEO;
pkt->stream_index = STREAM_MEDIA;
pkt->time_stamp = 0;
pkt->type_id = MSG_VIDEO;
RtmpCodec::inputRtmp(pkt);
}
}//namespace mediakit

78
ext-codec/H264Rtmp.h Normal file
View File

@@ -0,0 +1,78 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_H264RTMPCODEC_H
#define ZLMEDIAKIT_H264RTMPCODEC_H
#include "H264.h"
#include "Rtmp/RtmpCodec.h"
#include "Extension/Track.h"
namespace mediakit {
/**
* h264 Rtmp解码类
* 将 h264 over rtmp 解复用出 h264-Frame
*/
class H264RtmpDecoder : public RtmpCodec {
public:
using Ptr = std::shared_ptr<H264RtmpDecoder>;
H264RtmpDecoder(const Track::Ptr &track) : RtmpCodec(track) {}
/**
* 输入264 Rtmp包
* @param rtmp Rtmp包
*/
void inputRtmp(const RtmpPacket::Ptr &rtmp) override;
private:
void outputFrame(const char *data, size_t len, uint32_t dts, uint32_t pts);
void splitFrame(const uint8_t *data, size_t size, uint32_t dts, uint32_t pts);
};
/**
* 264 Rtmp打包类
*/
class H264RtmpEncoder : public RtmpCodec {
public:
using Ptr = std::shared_ptr<H264RtmpEncoder>;
/**
* 构造函数，track可以为空，此时则在inputFrame时输入sps pps
* 如果track不为空且包含sps pps信息，
* 那么inputFrame时可以不输入sps pps
* @param track
*/
H264RtmpEncoder(const Track::Ptr &track) : RtmpCodec(track) {}
/**
* 输入264帧，可以不带sps pps
* @param frame 帧数据
*/
bool inputFrame(const Frame::Ptr &frame) override;
/**
* 刷新输出所有frame缓存
*/
void flush() override;
/**
* 生成config包
*/
void makeConfigPacket() override;
private:
RtmpPacket::Ptr _rtmp_packet;
FrameMerger _merger { FrameMerger::mp4_nal_size };
};
}//namespace mediakit
#endif //ZLMEDIAKIT_H264RTMPCODEC_H

328
ext-codec/H264Rtp.cpp Normal file
View File

@@ -0,0 +1,328 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "H264Rtp.h"
#include "Common/config.h"
namespace mediakit{
#pragma pack(push, 1)
class FuFlags {
public:
#if __BYTE_ORDER == __BIG_ENDIAN
unsigned start_bit: 1;
unsigned end_bit: 1;
unsigned reserved: 1;
unsigned nal_type: 5;
#else
unsigned nal_type: 5;
unsigned reserved: 1;
unsigned end_bit: 1;
unsigned start_bit: 1;
#endif
};
#pragma pack(pop)
H264RtpDecoder::H264RtpDecoder() {
_frame = obtainFrame();
}
H264Frame::Ptr H264RtpDecoder::obtainFrame() {
auto frame = FrameImp::create<H264Frame>();
frame->_prefix_size = 4;
return frame;
}
bool H264RtpDecoder::inputRtp(const RtpPacket::Ptr &rtp, bool key_pos) {
auto seq = rtp->getSeq();
auto last_is_gop = _is_gop;
_is_gop = decodeRtp(rtp);
if (!_gop_dropped && seq != (uint16_t)(_last_seq + 1) && _last_seq) {
_gop_dropped = true;
WarnL << "start drop h264 gop, last seq:" << _last_seq << ", rtp:\r\n" << rtp->dumpString();
}
_last_seq = seq;
// 确保有sps rtp的时候，gop从sps开始；否则从关键帧开始
return _is_gop && !last_is_gop;
}
/*
RTF3984 5.2节 Common Structure of the RTP Payload Format
Table 1. Summary of NAL unit types and their payload structures
Type Packet Type name Section
---------------------------------------------------------
0 undefined -
1-23 NAL unit Single NAL unit packet per H.264 5.6
24 STAP-A Single-time aggregation packet 5.7.1
25 STAP-B Single-time aggregation packet 5.7.1
26 MTAP16 Multi-time aggregation packet 5.7.2
27 MTAP24 Multi-time aggregation packet 5.7.2
28 FU-A Fragmentation unit 5.8
29 FU-B Fragmentation unit 5.8
30-31 undefined -
*/
bool H264RtpDecoder::singleFrame(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp){
_frame->_buffer.assign("\x00\x00\x00\x01", 4);
_frame->_buffer.append((char *) ptr, size);
_frame->_pts = stamp;
auto key = _frame->keyFrame() || _frame->configFrame();
outputFrame(rtp, _frame);
return key;
}
bool H264RtpDecoder::unpackStapA(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp) {
//STAP-A 单一时间的组合包
auto have_key_frame = false;
auto end = ptr + size;
while (ptr + 2 < end) {
uint16_t len = (ptr[0] << 8) | ptr[1];
if (!len || ptr + len > end) {
WarnL << "invalid rtp data size:" << len << ",rtp:\r\n" << rtp->dumpString();
_gop_dropped = true;
break;
}
ptr += 2;
if (singleFrame(rtp, ptr, len, stamp)) {
have_key_frame = true;
}
ptr += len;
}
return have_key_frame;
}
bool H264RtpDecoder::mergeFu(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp, uint16_t seq){
auto nal_suffix = *ptr & (~0x1F);
FuFlags *fu = (FuFlags *) (ptr + 1);
if (fu->start_bit) {
//该帧的第一个rtp包
_frame->_buffer.assign("\x00\x00\x00\x01", 4);
_frame->_buffer.push_back(nal_suffix | fu->nal_type);
_frame->_pts = stamp;
_fu_dropped = false;
}
if (_fu_dropped) {
//该帧不完整
return false;
}
if (!fu->start_bit && seq != (uint16_t) (_last_seq + 1)) {
//中间的或末尾的rtp包，其seq必须连续，否则说明rtp丢包，那么该帧不完整，必须得丢弃
_fu_dropped = true;
_frame->_buffer.clear();
return false;
}
//后面追加数据
_frame->_buffer.append((char *) ptr + 2, size - 2);
if (!fu->end_bit) {
//非末尾包
return fu->start_bit ? (_frame->keyFrame() || _frame->configFrame()) : false;
}
//确保下一次fu必须收到第一个包
_fu_dropped = true;
//该帧最后一个rtp包,输出frame
outputFrame(rtp, _frame);
return false;
}
bool H264RtpDecoder::decodeRtp(const RtpPacket::Ptr &rtp) {
auto payload_size = rtp->getPayloadSize();
if (payload_size <= 0) {
//无实际负载
return false;
}
auto frame = rtp->getPayload();
auto stamp = rtp->getStampMS();
auto seq = rtp->getSeq();
int nal = H264_TYPE(frame[0]);
switch (nal) {
case 24:
// 24 STAP-A Single-time aggregation packet 5.7.1
return unpackStapA(rtp, frame + 1, payload_size - 1, stamp);
case 28:
// 28 FU-A Fragmentation unit
return mergeFu(rtp, frame, payload_size, stamp, seq);
default: {
if (nal < 24) {
//Single NAL Unit Packets
return singleFrame(rtp, frame, payload_size, stamp);
}
_gop_dropped = true;
WarnL << "不支持该类型的264 RTP包, nal type:" << nal << ", rtp:\r\n" << rtp->dumpString();
return false;
}
}
}
void H264RtpDecoder::outputFrame(const RtpPacket::Ptr &rtp, const H264Frame::Ptr &frame) {
if (frame->dropAble()) {
//不参与dts生成
frame->_dts = frame->_pts;
} else {
//rtsp没有dts，那么根据pts排序算法生成dts
_dts_generator.getDts(frame->_pts, frame->_dts);
}
if (frame->keyFrame() && _gop_dropped) {
_gop_dropped = false;
InfoL << "new gop received, rtp:\r\n" << rtp->dumpString();
}
if (!_gop_dropped) {
RtpCodec::inputFrame(frame);
}
_frame = obtainFrame();
}
////////////////////////////////////////////////////////////////////////
void H264RtpEncoder::insertConfigFrame(uint64_t pts){
if (!_sps || !_pps) {
return;
}
//gop缓存从sps开始，sps、pps后面还有时间戳相同的关键帧，所以mark bit为false
packRtp(_sps->data() + _sps->prefixSize(), _sps->size() - _sps->prefixSize(), pts, false, true);
packRtp(_pps->data() + _pps->prefixSize(), _pps->size() - _pps->prefixSize(), pts, false, false);
}
void H264RtpEncoder::packRtp(const char *ptr, size_t len, uint64_t pts, bool is_mark, bool gop_pos){
if (len + 3 <= getRtpInfo().getMaxSize()) {
// 采用STAP-A/Single NAL unit packet per H.264 模式
packRtpSmallFrame(ptr, len, pts, is_mark, gop_pos);
} else {
//STAP-A模式打包会大于MTU,所以采用FU-A模式
packRtpFu(ptr, len, pts, is_mark, gop_pos);
}
}
void H264RtpEncoder::packRtpFu(const char *ptr, size_t len, uint64_t pts, bool is_mark, bool gop_pos){
auto packet_size = getRtpInfo().getMaxSize() - 2;
if (len <= packet_size + 1) {
// 小于FU-A打包最小字节长度要求，采用STAP-A/Single NAL unit packet per H.264 模式
packRtpSmallFrame(ptr, len, pts, is_mark, gop_pos);
return;
}
//末尾5bit为nalu type，固定为28(FU-A)
auto fu_char_0 = (ptr[0] & (~0x1F)) | 28;
auto fu_char_1 = H264_TYPE(ptr[0]);
FuFlags *fu_flags = (FuFlags *) (&fu_char_1);
fu_flags->start_bit = 1;
size_t offset = 1;
while (!fu_flags->end_bit) {
if (!fu_flags->start_bit && len <= offset + packet_size) {
//FU-A end
packet_size = len - offset;
fu_flags->end_bit = 1;
}
//传入nullptr先不做payload的内存拷贝
auto rtp = getRtpInfo().makeRtp(TrackVideo, nullptr, packet_size + 2, fu_flags->end_bit && is_mark, pts);
//rtp payload 负载部分
uint8_t *payload = rtp->getPayload();
//FU-A 第1个字节
payload[0] = fu_char_0;
//FU-A 第2个字节
payload[1] = fu_char_1;
//H264 数据
memcpy(payload + 2, (uint8_t *) ptr + offset, packet_size);
//输入到rtp环形缓存
RtpCodec::inputRtp(rtp, gop_pos);
offset += packet_size;
fu_flags->start_bit = 0;
}
}
void H264RtpEncoder::packRtpSmallFrame(const char *data, size_t len, uint64_t pts, bool is_mark, bool gop_pos) {
GET_CONFIG(bool, h264_stap_a, Rtp::kH264StapA);
if (h264_stap_a) {
packRtpStapA(data, len, pts, is_mark, gop_pos);
} else {
packRtpSingleNalu(data, len, pts, is_mark, gop_pos);
}
}
void H264RtpEncoder::packRtpStapA(const char *ptr, size_t len, uint64_t pts, bool is_mark, bool gop_pos){
// 如果帧长度不超过mtu,为了兼容性 webrtc，采用STAP-A模式打包
auto rtp = getRtpInfo().makeRtp(TrackVideo, nullptr, len + 3, is_mark, pts);
uint8_t *payload = rtp->getPayload();
//STAP-A
payload[0] = (ptr[0] & (~0x1F)) | 24;
payload[1] = (len >> 8) & 0xFF;
payload[2] = len & 0xff;
memcpy(payload + 3, (uint8_t *) ptr, len);
RtpCodec::inputRtp(rtp, gop_pos);
}
void H264RtpEncoder::packRtpSingleNalu(const char *data, size_t len, uint64_t pts, bool is_mark, bool gop_pos) {
// Single NAL unit packet per H.264 模式
RtpCodec::inputRtp(getRtpInfo().makeRtp(TrackVideo, data, len, is_mark, pts), gop_pos);
}
bool H264RtpEncoder::inputFrame(const Frame::Ptr &frame) {
auto ptr = frame->data() + frame->prefixSize();
switch (H264_TYPE(ptr[0])) {
case H264Frame::NAL_SPS: {
_sps = Frame::getCacheAbleFrame(frame);
return true;
}
case H264Frame::NAL_PPS: {
_pps = Frame::getCacheAbleFrame(frame);
return true;
}
default: break;
}
GET_CONFIG(int,lowLatency,Rtp::kLowLatency);
if (lowLatency) { // 低延迟模式
if (_last_frame) {
flush();
}
inputFrame_l(frame, true);
} else {
if (_last_frame) {
//如果时间戳发生了变化，那么markbit才置true
inputFrame_l(_last_frame, _last_frame->pts() != frame->pts());
}
_last_frame = Frame::getCacheAbleFrame(frame);
}
return true;
}
void H264RtpEncoder::flush() {
if (_last_frame) {
// 如果时间戳发生了变化，那么markbit才置true
inputFrame_l(_last_frame, true);
_last_frame = nullptr;
}
}
bool H264RtpEncoder::inputFrame_l(const Frame::Ptr &frame, bool is_mark){
if (frame->keyFrame()) {
//保证每一个关键帧前都有SPS与PPS
insertConfigFrame(frame->pts());
}
packRtp(frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize(), frame->pts(), is_mark, false);
return true;
}
}//namespace mediakit

92
ext-codec/H264Rtp.h Normal file
View File

@@ -0,0 +1,92 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_H264RTPCODEC_H
#define ZLMEDIAKIT_H264RTPCODEC_H
#include "H264.h"
// for DtsGenerator
#include "Common/Stamp.h"
#include "Rtsp/RtpCodec.h"
namespace mediakit {
/**
* h264 rtp解码类
* 将 h264 over rtsp-rtp 解复用出 h264-Frame
* rfc3984
*/
class H264RtpDecoder : public RtpCodec{
public:
using Ptr = std::shared_ptr<H264RtpDecoder>;
H264RtpDecoder();
/**
* 输入264 rtp包
* @param rtp rtp包
* @param key_pos 此参数忽略之
*/
bool inputRtp(const RtpPacket::Ptr &rtp, bool key_pos = true) override;
private:
bool singleFrame(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp);
bool unpackStapA(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp);
bool mergeFu(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp, uint16_t seq);
bool decodeRtp(const RtpPacket::Ptr &rtp);
H264Frame::Ptr obtainFrame();
void outputFrame(const RtpPacket::Ptr &rtp, const H264Frame::Ptr &frame);
private:
bool _is_gop = false;
bool _gop_dropped = false;
bool _fu_dropped = true;
uint16_t _last_seq = 0;
H264Frame::Ptr _frame;
DtsGenerator _dts_generator;
};
/**
* 264 rtp打包类
*/
class H264RtpEncoder : public RtpCodec {
public:
using Ptr = std::shared_ptr<H264RtpEncoder>;
/**
* 输入264帧
* @param frame 帧数据，必须
*/
bool inputFrame(const Frame::Ptr &frame) override;
/**
* 刷新输出所有frame缓存
*/
void flush() override;
private:
void insertConfigFrame(uint64_t pts);
bool inputFrame_l(const Frame::Ptr &frame, bool is_mark);
void packRtp(const char *data, size_t len, uint64_t pts, bool is_mark, bool gop_pos);
void packRtpFu(const char *data, size_t len, uint64_t pts, bool is_mark, bool gop_pos);
void packRtpStapA(const char *data, size_t len, uint64_t pts, bool is_mark, bool gop_pos);
void packRtpSingleNalu(const char *data, size_t len, uint64_t pts, bool is_mark, bool gop_pos);
void packRtpSmallFrame(const char *data, size_t len, uint64_t pts, bool is_mark, bool gop_pos);
private:
Frame::Ptr _sps;
Frame::Ptr _pps;
Frame::Ptr _last_frame;
};
}//namespace mediakit{
#endif //ZLMEDIAKIT_H264RTPCODEC_H

325
ext-codec/H265.cpp Normal file
View File

@@ -0,0 +1,325 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "H265.h"
#include "H265Rtp.h"
#include "H265Rtmp.h"
#include "SPSParser.h"
#include "Util/base64.h"
#include "Common/Parser.h"
#include "Extension/Factory.h"
#ifdef ENABLE_MP4
#include "mpeg4-hevc.h"
#endif
using namespace std;
using namespace toolkit;
namespace mediakit {
bool getHEVCInfo(const char * vps, size_t vps_len,const char * sps,size_t sps_len,int &iVideoWidth, int &iVideoHeight, float &iVideoFps){
T_GetBitContext tGetBitBuf;
T_HEVCSPS tH265SpsInfo;
T_HEVCVPS tH265VpsInfo;
if ( vps_len > 2 ){
memset(&tGetBitBuf,0,sizeof(tGetBitBuf));
memset(&tH265VpsInfo,0,sizeof(tH265VpsInfo));
tGetBitBuf.pu8Buf = (uint8_t*)vps+2;
tGetBitBuf.iBufSize = (int)(vps_len-2);
if(0 != h265DecVideoParameterSet((void *) &tGetBitBuf, &tH265VpsInfo)){
return false;
}
}
if ( sps_len > 2 ){
memset(&tGetBitBuf,0,sizeof(tGetBitBuf));
memset(&tH265SpsInfo,0,sizeof(tH265SpsInfo));
tGetBitBuf.pu8Buf = (uint8_t*)sps+2;
tGetBitBuf.iBufSize = (int)(sps_len-2);
if(0 != h265DecSeqParameterSet((void *) &tGetBitBuf, &tH265SpsInfo)){
return false;
}
}
else
return false;
h265GetWidthHeight(&tH265SpsInfo, &iVideoWidth, &iVideoHeight);
iVideoFps = 0;
h265GeFramerate(&tH265VpsInfo, &tH265SpsInfo, &iVideoFps);
return true;
}
bool getHEVCInfo(const string &strVps, const string &strSps, int &iVideoWidth, int &iVideoHeight, float &iVideoFps) {
return getHEVCInfo(strVps.data(), strVps.size(), strSps.data(), strSps.size(), iVideoWidth, iVideoHeight,iVideoFps);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
H265Track::H265Track(const string &vps,const string &sps, const string &pps,int vps_prefix_len, int sps_prefix_len, int pps_prefix_len) {
_vps = vps.substr(vps_prefix_len);
_sps = sps.substr(sps_prefix_len);
_pps = pps.substr(pps_prefix_len);
update();
}
CodecId H265Track::getCodecId() const {
return CodecH265;
}
int H265Track::getVideoHeight() const {
return _height;
}
int H265Track::getVideoWidth() const {
return _width;
}
float H265Track::getVideoFps() const {
return _fps;
}
bool H265Track::ready() const {
return !_vps.empty() && !_sps.empty() && !_pps.empty();
}
bool H265Track::inputFrame(const Frame::Ptr &frame) {
int type = H265_TYPE(frame->data()[frame->prefixSize()]);
if (!frame->configFrame() && type != H265Frame::NAL_SEI_PREFIX && ready()) {
return inputFrame_l(frame);
}
bool ret = false;
splitH264(frame->data(), frame->size(), frame->prefixSize(), [&](const char *ptr, size_t len, size_t prefix) {
using H265FrameInternal = FrameInternal<H265FrameNoCacheAble>;
H265FrameInternal::Ptr sub_frame = std::make_shared<H265FrameInternal>(frame, (char *) ptr, len, prefix);
if (inputFrame_l(sub_frame)) {
ret = true;
}
});
return ret;
}
bool H265Track::inputFrame_l(const Frame::Ptr &frame) {
if (frame->keyFrame()) {
insertConfigFrame(frame);
_is_idr = true;
return VideoTrack::inputFrame(frame);
}
_is_idr = false;
bool ret = true;
//非idr帧
switch (H265_TYPE( frame->data()[frame->prefixSize()])) {
case H265Frame::NAL_VPS: {
_vps = string(frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize());
break;
}
case H265Frame::NAL_SPS: {
_sps = string(frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize());
break;
}
case H265Frame::NAL_PPS: {
_pps = string(frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize());
break;
}
default: {
ret = VideoTrack::inputFrame(frame);
break;
}
}
if (_width == 0 && ready()) {
update();
}
return ret;
}
toolkit::Buffer::Ptr H265Track::getExtraData() const {
CHECK(ready());
#ifdef ENABLE_MP4
struct mpeg4_hevc_t hevc;
memset(&hevc, 0, sizeof(hevc));
string vps_sps_pps = string("\x00\x00\x00\x01", 4) + _vps + string("\x00\x00\x00\x01", 4) + _sps + string("\x00\x00\x00\x01", 4) + _pps;
h265_annexbtomp4(&hevc, vps_sps_pps.data(), (int) vps_sps_pps.size(), NULL, 0, NULL, NULL);
std::string extra_data;
extra_data.resize(1024);
auto extra_data_size = mpeg4_hevc_decoder_configuration_record_save(&hevc, (uint8_t *)extra_data.data(), extra_data.size());
if (extra_data_size == -1) {
WarnL << "生成H265 extra_data 失败";
return nullptr;
}
return std::make_shared<BufferString>(std::move(extra_data));
#else
WarnL << "请开启MP4相关功能并使能\"ENABLE_MP4\",否则对H265的支持不完善";
return nullptr;
#endif
}
void H265Track::setExtraData(const uint8_t *data, size_t bytes) {
#ifdef ENABLE_MP4
struct mpeg4_hevc_t hevc;
memset(&hevc, 0, sizeof(hevc));
if (mpeg4_hevc_decoder_configuration_record_load(data, bytes, &hevc) > 0) {
std::vector<uint8_t> config(bytes * 2);
int size = mpeg4_hevc_to_nalu(&hevc, config.data(), bytes * 2);
if (size > 4) {
splitH264((char *)config.data(), size, 4, [&](const char *ptr, size_t len, size_t prefix) {
inputFrame_l(std::make_shared<H265FrameNoCacheAble>((char *)ptr, len, 0, 0, prefix));
});
update();
}
}
#else
WarnL << "请开启MP4相关功能并使能\"ENABLE_MP4\",否则对H265的支持不完善";
#endif
}
bool H265Track::update() {
return getHEVCInfo(_vps, _sps, _width, _height, _fps);
}
Track::Ptr H265Track::clone() const {
return std::make_shared<H265Track>(*this);
}
void H265Track::insertConfigFrame(const Frame::Ptr &frame) {
if (_is_idr) {
return;
}
if (!_vps.empty()) {
auto vpsFrame = FrameImp::create<H265Frame>();
vpsFrame->_prefix_size = 4;
vpsFrame->_buffer.assign("\x00\x00\x00\x01", 4);
vpsFrame->_buffer.append(_vps);
vpsFrame->_dts = frame->dts();
vpsFrame->setIndex(frame->getIndex());
VideoTrack::inputFrame(vpsFrame);
}
if (!_sps.empty()) {
auto spsFrame = FrameImp::create<H265Frame>();
spsFrame->_prefix_size = 4;
spsFrame->_buffer.assign("\x00\x00\x00\x01", 4);
spsFrame->_buffer.append(_sps);
spsFrame->_dts = frame->dts();
spsFrame->setIndex(frame->getIndex());
VideoTrack::inputFrame(spsFrame);
}
if (!_pps.empty()) {
auto ppsFrame = FrameImp::create<H265Frame>();
ppsFrame->_prefix_size = 4;
ppsFrame->_buffer.assign("\x00\x00\x00\x01", 4);
ppsFrame->_buffer.append(_pps);
ppsFrame->_dts = frame->dts();
ppsFrame->setIndex(frame->getIndex());
VideoTrack::inputFrame(ppsFrame);
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/**
* h265类型sdp
*/
class H265Sdp : public Sdp {
public:
/**
* 构造函数
* @param sps 265 sps,不带0x00000001头
* @param pps 265 pps,不带0x00000001头
* @param payload_type rtp payload type 默认96
* @param bitrate 比特率
*/
H265Sdp(const string &strVPS, const string &strSPS, const string &strPPS, int payload_type, int bitrate) : Sdp(90000, payload_type) {
//视频通道
_printer << "m=video 0 RTP/AVP " << payload_type << "\r\n";
if (bitrate) {
_printer << "b=AS:" << bitrate << "\r\n";
}
_printer << "a=rtpmap:" << payload_type << " " << getCodecName(CodecH265) << "/" << 90000 << "\r\n";
_printer << "a=fmtp:" << payload_type << " ";
_printer << "sprop-vps=";
_printer << encodeBase64(strVPS) << "; ";
_printer << "sprop-sps=";
_printer << encodeBase64(strSPS) << "; ";
_printer << "sprop-pps=";
_printer << encodeBase64(strPPS) << "\r\n";
}
string getSdp() const override { return _printer; }
private:
_StrPrinter _printer;
};
Sdp::Ptr H265Track::getSdp(uint8_t payload_type) const {
if (!ready()) {
WarnL << getCodecName() << " Track未准备好";
return nullptr;
}
return std::make_shared<H265Sdp>(_vps, _sps, _pps, payload_type, getBitRate() / 1024);
}
namespace {
CodecId getCodec() {
return CodecH265;
}
Track::Ptr getTrackByCodecId(int sample_rate, int channels, int sample_bit) {
return std::make_shared<H265Track>();
}
Track::Ptr getTrackBySdp(const SdpTrack::Ptr &track) {
// a=fmtp:96 sprop-sps=QgEBAWAAAAMAsAAAAwAAAwBdoAKAgC0WNrkky/AIAAADAAgAAAMBlQg=; sprop-pps=RAHA8vA8kAA=
auto map = Parser::parseArgs(track->_fmtp, ";", "=");
auto vps = decodeBase64(map["sprop-vps"]);
auto sps = decodeBase64(map["sprop-sps"]);
auto pps = decodeBase64(map["sprop-pps"]);
if (sps.empty() || pps.empty()) {
// 如果sdp里面没有sps/pps,那么可能在后续的rtp里面恢复出sps/pps
return std::make_shared<H265Track>();
}
return std::make_shared<H265Track>(vps, sps, pps, 0, 0, 0);
}
RtpCodec::Ptr getRtpEncoderByCodecId(uint8_t pt) {
return std::make_shared<H265RtpEncoder>();
}
RtpCodec::Ptr getRtpDecoderByCodecId() {
return std::make_shared<H265RtpDecoder>();
}
RtmpCodec::Ptr getRtmpEncoderByTrack(const Track::Ptr &track) {
return std::make_shared<H265RtmpEncoder>(track);
}
RtmpCodec::Ptr getRtmpDecoderByTrack(const Track::Ptr &track) {
return std::make_shared<H265RtmpDecoder>(track);
}
Frame::Ptr getFrameFromPtr(const char *data, size_t bytes, uint64_t dts, uint64_t pts) {
return std::make_shared<H265FrameNoCacheAble>((char *)data, bytes, dts, pts, prefixSize(data, bytes));
}
} // namespace
CodecPlugin h265_plugin = { getCodec,
getTrackByCodecId,
getTrackBySdp,
getRtpEncoderByCodecId,
getRtpDecoderByCodecId,
getRtmpEncoderByTrack,
getRtmpDecoderByTrack,
getFrameFromPtr };
}//namespace mediakit

163
ext-codec/H265.h Normal file
View File

@@ -0,0 +1,163 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_H265_H
#define ZLMEDIAKIT_H265_H
#include "H264.h"
#include "Extension/Track.h"
#include "Extension/Frame.h"
#define H265_TYPE(v) (((uint8_t)(v) >> 1) & 0x3f)
namespace mediakit {
template<typename Parent>
class H265FrameHelper : public Parent{
public:
friend class FrameImp;
friend class toolkit::ResourcePool_l<H265FrameHelper>;
using Ptr = std::shared_ptr<H265FrameHelper>;
enum {
NAL_TRAIL_N = 0,
NAL_TRAIL_R = 1,
NAL_TSA_N = 2,
NAL_TSA_R = 3,
NAL_STSA_N = 4,
NAL_STSA_R = 5,
NAL_RADL_N = 6,
NAL_RADL_R = 7,
NAL_RASL_N = 8,
NAL_RASL_R = 9,
NAL_BLA_W_LP = 16,
NAL_BLA_W_RADL = 17,
NAL_BLA_N_LP = 18,
NAL_IDR_W_RADL = 19,
NAL_IDR_N_LP = 20,
NAL_CRA_NUT = 21,
NAL_RSV_IRAP_VCL22 = 22,
NAL_RSV_IRAP_VCL23 = 23,
NAL_VPS = 32,
NAL_SPS = 33,
NAL_PPS = 34,
NAL_AUD = 35,
NAL_EOS_NUT = 36,
NAL_EOB_NUT = 37,
NAL_FD_NUT = 38,
NAL_SEI_PREFIX = 39,
NAL_SEI_SUFFIX = 40,
};
template<typename ...ARGS>
H265FrameHelper(ARGS &&...args): Parent(std::forward<ARGS>(args)...) {
this->_codec_id = CodecH265;
}
bool keyFrame() const override {
auto nal_ptr = (uint8_t *) this->data() + this->prefixSize();
auto type = H265_TYPE(*nal_ptr);
// 参考自FFmpeg: IRAP VCL NAL unit types span the range
// [BLA_W_LP (16), RSV_IRAP_VCL23 (23)].
return (type >= NAL_BLA_W_LP && type <= NAL_RSV_IRAP_VCL23) && decodeAble() ;
}
bool configFrame() const override {
auto nal_ptr = (uint8_t *) this->data() + this->prefixSize();
switch (H265_TYPE(*nal_ptr)) {
case NAL_VPS:
case NAL_SPS:
case NAL_PPS : return true;
default : return false;
}
}
bool dropAble() const override {
auto nal_ptr = (uint8_t *) this->data() + this->prefixSize();
switch (H265_TYPE(*nal_ptr)) {
case NAL_AUD:
case NAL_SEI_SUFFIX:
case NAL_SEI_PREFIX: return true;
default: return false;
}
}
bool decodeAble() const override {
auto nal_ptr = (uint8_t *) this->data() + this->prefixSize();
auto type = H265_TYPE(*nal_ptr);
//多slice情况下, first_slice_segment_in_pic_flag 表示其为一帧的开始
return type >= NAL_TRAIL_N && type <= NAL_RSV_IRAP_VCL23 && (nal_ptr[2] & 0x80);
}
};
/**
* 265帧类
*/
using H265Frame = H265FrameHelper<FrameImp>;
/**
* 防止内存拷贝的H265类
* 用户可以通过该类型快速把一个指针无拷贝的包装成Frame类
*/
using H265FrameNoCacheAble = H265FrameHelper<FrameFromPtr>;
/**
* 265视频通道
*/
class H265Track : public VideoTrack {
public:
using Ptr = std::shared_ptr<H265Track>;
/**
* 不指定sps pps构造h265类型的媒体
* 在随后的inputFrame中获取sps pps
*/
H265Track() = default;
/**
* 构造h265类型的媒体
* @param vps vps帧数据
* @param sps sps帧数据
* @param pps pps帧数据
* @param vps_prefix_len 265头长度，可以为3个或4个字节，一般为0x00 00 00 01
* @param sps_prefix_len 265头长度，可以为3个或4个字节，一般为0x00 00 00 01
* @param pps_prefix_len 265头长度，可以为3个或4个字节，一般为0x00 00 00 01
*/
H265Track(const std::string &vps,const std::string &sps, const std::string &pps,int vps_prefix_len = 4, int sps_prefix_len = 4, int pps_prefix_len = 4);
bool ready() const override;
CodecId getCodecId() const override;
int getVideoWidth() const override;
int getVideoHeight() const override;
float getVideoFps() const override;
bool inputFrame(const Frame::Ptr &frame) override;
toolkit::Buffer::Ptr getExtraData() const override;
void setExtraData(const uint8_t *data, size_t size) override;
bool update() override;
private:
Sdp::Ptr getSdp(uint8_t payload_type) const override;
Track::Ptr clone() const override;
bool inputFrame_l(const Frame::Ptr &frame);
void insertConfigFrame(const Frame::Ptr &frame);
private:
bool _is_idr = false;
int _width = 0;
int _height = 0;
float _fps = 0;
std::string _vps;
std::string _sps;
std::string _pps;
};
}//namespace mediakit
#endif //ZLMEDIAKIT_H265_H

174
ext-codec/H265Rtmp.cpp Normal file
View File

@@ -0,0 +1,174 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "H265Rtmp.h"
#include "Rtmp/utils.h"
#include "Common/config.h"
#ifdef ENABLE_MP4
#include "mpeg4-hevc.h"
#endif // ENABLE_MP4
using namespace std;
using namespace toolkit;
namespace mediakit {
void H265RtmpDecoder::inputRtmp(const RtmpPacket::Ptr &pkt) {
if (_info.codec == CodecInvalid) {
// 先判断是否为增强型rtmp
parseVideoRtmpPacket((uint8_t *)pkt->data(), pkt->size(), &_info);
}
if (_info.is_enhanced) {
// 增强型rtmp
parseVideoRtmpPacket((uint8_t *)pkt->data(), pkt->size(), &_info);
if (!_info.is_enhanced || _info.codec != CodecH265) {
throw std::invalid_argument("Invalid enhanced-rtmp hevc packet!");
}
switch (_info.video.pkt_type) {
case RtmpPacketType::PacketTypeSequenceStart: {
getTrack()->setExtraData((uint8_t *)pkt->data() + RtmpPacketInfo::kEnhancedRtmpHeaderSize, pkt->size() - RtmpPacketInfo::kEnhancedRtmpHeaderSize);
break;
}
case RtmpPacketType::PacketTypeCodedFramesX:
case RtmpPacketType::PacketTypeCodedFrames: {
auto data = (uint8_t *)pkt->data() + RtmpPacketInfo::kEnhancedRtmpHeaderSize;
auto size = pkt->size() - RtmpPacketInfo::kEnhancedRtmpHeaderSize;
auto pts = pkt->time_stamp;
CHECK(size > 3);
if (RtmpPacketType::PacketTypeCodedFrames == _info.video.pkt_type) {
// SI24 = [CompositionTime Offset]
int32_t cts = (((data[0] << 16) | (data[1] << 8) | (data[2])) + 0xff800000) ^ 0xff800000;
pts += cts;
data += 3;
size -= 3;
}
CHECK(size > 4);
splitFrame(data, size, pkt->time_stamp, pts);
break;
}
default: WarnL << "Unknown pkt_type: " << (int)_info.video.pkt_type; break;
}
return;
}
// 国内扩展(12) H265 rtmp
if (pkt->isConfigFrame()) {
CHECK(pkt->size() > 5);
getTrack()->setExtraData((uint8_t *)pkt->data() + 5, pkt->size() - 5);
return;
}
CHECK(pkt->size() > 9);
uint8_t *cts_ptr = (uint8_t *)(pkt->buffer.data() + 2);
int32_t cts = (((cts_ptr[0] << 16) | (cts_ptr[1] << 8) | (cts_ptr[2])) + 0xff800000) ^ 0xff800000;
auto pts = pkt->time_stamp + cts;
splitFrame((uint8_t *)pkt->data() + 5, pkt->size() - 5, pkt->time_stamp, pts);
}
void H265RtmpDecoder::splitFrame(const uint8_t *data, size_t size, uint32_t dts, uint32_t pts) {
auto end = data + size;
while (data + 4 < end) {
uint32_t frame_len = load_be32(data);
data += 4;
if (data + frame_len > end) {
break;
}
outputFrame((const char *)data, frame_len, dts, pts);
data += frame_len;
}
}
inline void H265RtmpDecoder::outputFrame(const char *data, size_t size, uint32_t dts, uint32_t pts) {
auto frame = FrameImp::create<H265Frame>();
frame->_prefix_size = 4;
frame->_dts = dts;
frame->_pts = pts;
frame->_buffer.assign("\x00\x00\x00\x01", 4); // 添加265头
frame->_buffer.append(data, size);
RtmpCodec::inputFrame(frame);
}
////////////////////////////////////////////////////////////////////////
void H265RtmpEncoder::flush() {
inputFrame(nullptr);
}
bool H265RtmpEncoder::inputFrame(const Frame::Ptr &frame) {
if (!_rtmp_packet) {
_rtmp_packet = RtmpPacket::create();
GET_CONFIG(bool, enhanced, Rtmp::kEnhanced);
_rtmp_packet->buffer.resize((enhanced ? RtmpPacketInfo::kEnhancedRtmpHeaderSize : 2) + 3);
}
return _merger.inputFrame(frame, [this](uint64_t dts, uint64_t pts, const Buffer::Ptr &, bool have_key_frame) {
GET_CONFIG(bool, enhanced, Rtmp::kEnhanced);
if (enhanced) {
auto header = (RtmpVideoHeaderEnhanced *)_rtmp_packet->data();
header->enhanced = 1;
header->pkt_type = (int)RtmpPacketType::PacketTypeCodedFrames;
header->frame_type = have_key_frame ? (int)RtmpFrameType::key_frame : (int)RtmpFrameType::inter_frame;
header->fourcc = htonl((uint32_t)RtmpVideoCodec::fourcc_hevc);
} else {
// flags
_rtmp_packet->buffer[0] = (uint8_t)RtmpVideoCodec::h265 | ((uint8_t)(have_key_frame ? RtmpFrameType::key_frame : RtmpFrameType::inter_frame) << 4);
_rtmp_packet->buffer[1] = (uint8_t)RtmpH264PacketType::h264_nalu;
}
int32_t cts = pts - dts;
// cts
set_be24(&_rtmp_packet->buffer[enhanced ? 5 : 2], cts);
_rtmp_packet->time_stamp = dts;
_rtmp_packet->body_size = _rtmp_packet->buffer.size();
_rtmp_packet->chunk_id = CHUNK_VIDEO;
_rtmp_packet->stream_index = STREAM_MEDIA;
_rtmp_packet->type_id = MSG_VIDEO;
// 输出rtmp packet
RtmpCodec::inputRtmp(_rtmp_packet);
_rtmp_packet = nullptr;
}, &_rtmp_packet->buffer);
}
void H265RtmpEncoder::makeConfigPacket() {
auto pkt = RtmpPacket::create();
GET_CONFIG(bool, enhanced, Rtmp::kEnhanced);
if (enhanced) {
pkt->buffer.resize(RtmpPacketInfo::kEnhancedRtmpHeaderSize);
auto header = (RtmpVideoHeaderEnhanced *)pkt->data();
header->enhanced = 1;
header->pkt_type = (int)RtmpPacketType::PacketTypeSequenceStart;
header->frame_type = (int)RtmpFrameType::key_frame;
header->fourcc = htonl((uint32_t)RtmpVideoCodec::fourcc_hevc);
} else {
auto flags = (uint8_t)RtmpVideoCodec::h265;
flags |= ((uint8_t)RtmpFrameType::key_frame << 4);
// header
pkt->buffer.push_back(flags);
pkt->buffer.push_back((uint8_t)RtmpH264PacketType::h264_config_header);
// cts
pkt->buffer.append("\x0\x0\x0", 3);
}
// HEVCDecoderConfigurationRecord
auto extra_data = getTrack()->getExtraData();
CHECK(extra_data);
pkt->buffer.append(extra_data->data(), extra_data->size());
pkt->body_size = pkt->buffer.size();
pkt->chunk_id = CHUNK_VIDEO;
pkt->stream_index = STREAM_MEDIA;
pkt->time_stamp = 0;
pkt->type_id = MSG_VIDEO;
RtmpCodec::inputRtmp(pkt);
}
} // namespace mediakit

81
ext-codec/H265Rtmp.h Normal file
View File

@@ -0,0 +1,81 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_H265RTMPCODEC_H
#define ZLMEDIAKIT_H265RTMPCODEC_H
#include "H265.h"
#include "Rtmp/RtmpCodec.h"
#include "Extension/Track.h"
namespace mediakit {
/**
* h265 Rtmp解码类
* 将 h265 over rtmp 解复用出 h265-Frame
*/
class H265RtmpDecoder : public RtmpCodec {
public:
using Ptr = std::shared_ptr<H265RtmpDecoder>;
H265RtmpDecoder(const Track::Ptr &track) : RtmpCodec(track) {}
/**
* 输入265 Rtmp包
* @param rtmp Rtmp包
*/
void inputRtmp(const RtmpPacket::Ptr &rtmp) override;
protected:
void outputFrame(const char *data, size_t size, uint32_t dts, uint32_t pts);
void splitFrame(const uint8_t *data, size_t size, uint32_t dts, uint32_t pts);
protected:
RtmpPacketInfo _info;
};
/**
* 265 Rtmp打包类
*/
class H265RtmpEncoder : public RtmpCodec {
public:
using Ptr = std::shared_ptr<H265RtmpEncoder>;
/**
* 构造函数，track可以为空，此时则在inputFrame时输入sps pps
* 如果track不为空且包含sps pps信息，
* 那么inputFrame时可以不输入sps pps
* @param track
*/
H265RtmpEncoder(const Track::Ptr &track) : RtmpCodec(track) {}
/**
* 输入265帧，可以不带sps pps
* @param frame 帧数据
*/
bool inputFrame(const Frame::Ptr &frame) override;
/**
* 刷新输出所有frame缓存
*/
void flush() override;
/**
* 生成config包
*/
void makeConfigPacket() override;
private:
RtmpPacket::Ptr _rtmp_packet;
FrameMerger _merger { FrameMerger::mp4_nal_size };
};
} // namespace mediakit
#endif // ZLMEDIAKIT_H265RTMPCODEC_H

366
ext-codec/H265Rtp.cpp Normal file
View File

@@ -0,0 +1,366 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "H265Rtp.h"
#include "Common/config.h"
namespace mediakit{
//https://datatracker.ietf.org/doc/rfc7798/
//H265 nalu 头两个字节的定义
/*
0 1
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|F| Type | LayerId | TID |
+-------------+-----------------+
Forbidden zero(F) : 1 bit
NAL unit type(Type) : 6 bits
NUH layer ID(LayerId) : 6 bits
NUH temporal ID plus 1 (TID) : 3 bits
*/
H265RtpDecoder::H265RtpDecoder() {
_frame = obtainFrame();
}
H265Frame::Ptr H265RtpDecoder::obtainFrame() {
auto frame = FrameImp::create<H265Frame>();
frame->_prefix_size = 4;
return frame;
}
#define AV_RB16(x) \
((((const uint8_t*)(x))[0] << 8) | \
((const uint8_t*)(x))[1])
#define CHECK_SIZE(total, size, ret) \
if (total < size) { \
WarnL << "invalid rtp data size:" << total << " < " << size << ",rtp:\r\n" << rtp->dumpString(); _gop_dropped = true; return ret; \
}
// 4.4.2. Aggregation Packets (APs) (p25)
/*
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| RTP Header |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| PayloadHdr (Type=48) | NALU 1 DONL |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| NALU 1 Size | NALU 1 HDR |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
| NALU 1 Data . . . |
| |
+ . . . +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| | NALU 2 DOND | NALU 2 Size |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| NALU 2 HDR | |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ NALU 2 Data |
| |
| . . . +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| : ...OPTIONAL RTP padding |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
*/
bool H265RtpDecoder::unpackAp(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp){
bool have_key_frame = false;
//忽略PayloadHdr
CHECK_SIZE(size, 2, have_key_frame);
ptr += 2;
size -= 2;
while (size) {
if (_using_donl_field) {
CHECK_SIZE(size, 2, have_key_frame);
uint16_t donl = AV_RB16(ptr);
size -= 2;
ptr += 2;
}
CHECK_SIZE(size, 2, have_key_frame);
uint16_t nalu_size = AV_RB16(ptr);
size -= 2;
ptr += 2;
CHECK_SIZE(size, nalu_size, have_key_frame)
if (singleFrame(rtp, ptr, nalu_size, stamp)) {
have_key_frame = true;
}
size -= nalu_size;
ptr += nalu_size;
}
return have_key_frame;
}
// 4.4.3. Fragmentation Units (p29)
/*
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| PayloadHdr (Type=49) | FU header | DONL (cond) |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-|
| DONL (cond) | |
|-+-+-+-+-+-+-+-+ |
| FU payload |
| |
| +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| : ...OPTIONAL RTP padding |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+---------------+
|0|1|2|3|4|5|6|7|
+-+-+-+-+-+-+-+-+
|S|E| FuType |
+---------------+
*/
bool H265RtpDecoder::mergeFu(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp, uint16_t seq){
CHECK_SIZE(size, 4, false);
auto s_bit = ptr[2] >> 7;
auto e_bit = (ptr[2] >> 6) & 0x01;
auto type = ptr[2] & 0x3f;
if (s_bit) {
//该帧的第一个rtp包
_frame->_buffer.assign("\x00\x00\x00\x01", 4);
_frame->_buffer.push_back((type << 1) | (ptr[0] & 0x81));
_frame->_buffer.push_back(ptr[1]);
_frame->_pts = stamp;
_fu_dropped = false;
}
if (_fu_dropped) {
//该帧不完整
return false;
}
if (!s_bit && seq != (uint16_t) (_last_seq + 1)) {
//中间的或末尾的rtp包，其seq必须连续，否则说明rtp丢包，那么该帧不完整，必须得丢弃
_fu_dropped = true;
_frame->_buffer.clear();
return false;
}
//跳过PayloadHdr + FU header
ptr += 3;
size -= 3;
if (_using_donl_field) {
//DONL确保不少于2个字节
CHECK_SIZE(size, 2, false);
uint16_t donl = AV_RB16(ptr);
size -= 2;
ptr += 2;
}
CHECK_SIZE(size, 1, false);
//后面追加数据
_frame->_buffer.append((char *) ptr, size);
if (!e_bit) {
//非末尾包
return s_bit ? (_frame->keyFrame() || _frame->configFrame()) : false;
}
//确保下一次fu必须收到第一个包
_fu_dropped = true;
//该帧最后一个rtp包
outputFrame(rtp, _frame);
return false;
}
bool H265RtpDecoder::inputRtp(const RtpPacket::Ptr &rtp, bool) {
auto seq = rtp->getSeq();
auto last_is_gop = _is_gop;
_is_gop = decodeRtp(rtp);
if (!_gop_dropped && seq != (uint16_t) (_last_seq + 1) && _last_seq) {
_gop_dropped = true;
WarnL << "start drop h265 gop, last seq:" << _last_seq << ", rtp:\r\n" << rtp->dumpString();
}
_last_seq = seq;
// 确保有sps rtp的时候，gop从sps开始；否则从关键帧开始
return _is_gop && !last_is_gop;
}
bool H265RtpDecoder::decodeRtp(const RtpPacket::Ptr &rtp) {
auto payload_size = rtp->getPayloadSize();
if (payload_size <= 0) {
//无实际负载
return false;
}
auto frame = rtp->getPayload();
auto stamp = rtp->getStampMS();
auto seq = rtp->getSeq();
int nal = H265_TYPE(frame[0]);
switch (nal) {
case 48:
// aggregated packet (AP) - with two or more NAL units
return unpackAp(rtp, frame, payload_size, stamp);
case 49:
// fragmentation unit (FU)
return mergeFu(rtp, frame, payload_size, stamp, seq);
default: {
if (nal < 48) {
// Single NAL Unit Packets (p24)
return singleFrame(rtp, frame, payload_size, stamp);
}
_gop_dropped = true;
WarnL << "不支持该类型的265 RTP包, nal type" << nal << ", rtp:\r\n" << rtp->dumpString();
return false;
}
}
}
bool H265RtpDecoder::singleFrame(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp){
_frame->_buffer.assign("\x00\x00\x00\x01", 4);
_frame->_buffer.append((char *) ptr, size);
_frame->_pts = stamp;
auto key = _frame->keyFrame() || _frame->configFrame();
outputFrame(rtp, _frame);
return key;
}
void H265RtpDecoder::outputFrame(const RtpPacket::Ptr &rtp, const H265Frame::Ptr &frame) {
if (frame->dropAble()) {
//不参与dts生成
frame->_dts = frame->_pts;
} else {
//rtsp没有dts，那么根据pts排序算法生成dts
_dts_generator.getDts(frame->_pts, frame->_dts);
}
if (frame->keyFrame() && _gop_dropped) {
_gop_dropped = false;
InfoL << "new gop received, rtp:\r\n" << rtp->dumpString();
}
if (!_gop_dropped) {
RtpCodec::inputFrame(frame);
}
_frame = obtainFrame();
}
////////////////////////////////////////////////////////////////////////
void H265RtpEncoder::packRtpFu(const char *ptr, size_t len, uint64_t pts, bool is_mark, bool gop_pos){
auto max_size = getRtpInfo().getMaxSize() - 3;
auto nal_type = H265_TYPE(ptr[0]); //获取NALU的5bit 帧类型
unsigned char s_e_flags;
bool fu_start = true;
bool mark_bit = false;
size_t offset = 2;
while (!mark_bit) {
if (len <= offset + max_size) {
// FU end
mark_bit = true;
max_size = len - offset;
s_e_flags = (1 << 6) | nal_type;
} else if (fu_start) {
// FU start
s_e_flags = (1 << 7) | nal_type;
} else {
// FU mid
s_e_flags = nal_type;
}
{
// 传入nullptr先不做payload的内存拷贝
auto rtp = getRtpInfo().makeRtp(TrackVideo, nullptr, max_size + 3, mark_bit, pts);
// rtp payload 负载部分
uint8_t *payload = rtp->getPayload();
// FU 第1个字节，表明为FU
payload[0] = 49 << 1;
// FU 第2个字节貌似固定为1
payload[1] = ptr[1]; // 1;
// FU 第3个字节
payload[2] = s_e_flags;
// H265 数据
memcpy(payload + 3, ptr + offset, max_size);
// 输入到rtp环形缓存
RtpCodec::inputRtp(rtp, fu_start && gop_pos);
}
offset += max_size;
fu_start = false;
}
}
void H265RtpEncoder::packRtp(const char *ptr, size_t len, uint64_t pts, bool is_mark, bool gop_pos){
if (len <= getRtpInfo().getMaxSize()) {
//signal-nalu
RtpCodec::inputRtp(getRtpInfo().makeRtp(TrackVideo, ptr, len, is_mark, pts), gop_pos);
} else {
//FU-A模式
packRtpFu(ptr, len, pts, is_mark, gop_pos);
}
}
void H265RtpEncoder::insertConfigFrame(uint64_t pts){
if (!_sps || !_pps || !_vps) {
WarnL<<" not ok";
return;
}
//gop缓存从vps 开始，vps ,sps、pps后面还有时间戳相同的关键帧，所以mark bit为false
packRtp(_vps->data() + _vps->prefixSize(), _vps->size() - _vps->prefixSize(), pts, false, true);
packRtp(_sps->data() + _sps->prefixSize(), _sps->size() - _sps->prefixSize(), pts, false, false);
packRtp(_pps->data() + _pps->prefixSize(), _pps->size() - _pps->prefixSize(), pts, false, false);
}
bool H265RtpEncoder::inputFrame_l(const Frame::Ptr &frame, bool is_mark){
if (frame->keyFrame()) {
//保证每一个关键帧前都有SPS PPS VPS
insertConfigFrame(frame->pts());
}
packRtp(frame->data() + frame->prefixSize(), frame->size() - frame->prefixSize(), frame->pts(), is_mark, false);
return true;
}
bool H265RtpEncoder::inputFrame(const Frame::Ptr &frame) {
auto ptr = (uint8_t *) frame->data() + frame->prefixSize();
auto nal_type = H265_TYPE(ptr[0]); //获取NALU的5bit 帧类型
switch (nal_type) {
case H265Frame::NAL_SPS: {
_sps = Frame::getCacheAbleFrame(frame);
return true;
}
case H265Frame::NAL_PPS: {
_pps = Frame::getCacheAbleFrame(frame);
return true;
}
case H265Frame::NAL_VPS:{
_vps = Frame::getCacheAbleFrame(frame);
return true;
}
default: break;
}
GET_CONFIG(int,lowLatency,Rtp::kLowLatency);
if (lowLatency) { // 低延迟模式
if (_last_frame) {
flush();
}
inputFrame_l(frame, true);
} else {
if (_last_frame) {
//如果时间戳发生了变化，那么markbit才置true
inputFrame_l(_last_frame, _last_frame->pts() != frame->pts());
}
_last_frame = Frame::getCacheAbleFrame(frame);
}
return true;
}
void H265RtpEncoder::flush() {
if (_last_frame) {
// 如果时间戳发生了变化，那么markbit才置true
inputFrame_l(_last_frame, true);
_last_frame = nullptr;
}
}
}//namespace mediakit

90
ext-codec/H265Rtp.h Normal file
View File

@@ -0,0 +1,90 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_H265RTPCODEC_H
#define ZLMEDIAKIT_H265RTPCODEC_H
#include "H265.h"
#include "Rtsp/RtpCodec.h"
// for DtsGenerator
#include "Common/Stamp.h"
namespace mediakit {
/**
* h265 rtp解码类
* 将 h265 over rtsp-rtp 解复用出 h265-Frame
* 《草案（H265-over-RTP）draft-ietf-payload-rtp-h265-07.pdf》
*/
class H265RtpDecoder : public RtpCodec {
public:
using Ptr = std::shared_ptr<H265RtpDecoder>;
H265RtpDecoder();
/**
* 输入265 rtp包
* @param rtp rtp包
* @param key_pos 此参数忽略之
*/
bool inputRtp(const RtpPacket::Ptr &rtp, bool key_pos = true) override;
private:
bool unpackAp(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp);
bool mergeFu(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp, uint16_t seq);
bool singleFrame(const RtpPacket::Ptr &rtp, const uint8_t *ptr, ssize_t size, uint64_t stamp);
bool decodeRtp(const RtpPacket::Ptr &rtp);
H265Frame::Ptr obtainFrame();
void outputFrame(const RtpPacket::Ptr &rtp, const H265Frame::Ptr &frame);
private:
bool _is_gop = false;
bool _using_donl_field = false;
bool _gop_dropped = false;
bool _fu_dropped = true;
uint16_t _last_seq = 0;
H265Frame::Ptr _frame;
DtsGenerator _dts_generator;
};
/**
* 265 rtp打包类
*/
class H265RtpEncoder : public RtpCodec {
public:
using Ptr = std::shared_ptr<H265RtpEncoder>;
/**
* 输入265帧
* @param frame 帧数据，必须
*/
bool inputFrame(const Frame::Ptr &frame) override;
/**
* 刷新输出所有frame缓存
*/
void flush() override;
private:
void packRtp(const char *ptr, size_t len, uint64_t pts, bool is_mark, bool gop_pos);
void packRtpFu(const char *ptr, size_t len, uint64_t pts, bool is_mark, bool gop_pos);
void insertConfigFrame(uint64_t pts);
bool inputFrame_l(const Frame::Ptr &frame, bool is_mark);
private:
Frame::Ptr _sps;
Frame::Ptr _pps;
Frame::Ptr _vps;
Frame::Ptr _last_frame;
};
}//namespace mediakit{
#endif //ZLMEDIAKIT_H265RTPCODEC_H

101
ext-codec/JPEG.cpp Normal file
View File

@@ -0,0 +1,101 @@
#include "JPEG.h"
#include "JPEGRtp.h"
#include "Rtsp/Rtsp.h"
#include "Util/util.h"
#include "Extension/Factory.h"
using namespace toolkit;
namespace mediakit {
bool JPEGTrack::inputFrame(const Frame::Ptr &frame) {
if (!ready()) {
if (_height > 0 && _width > 0) {
if (_tmp == 0) _tmp = frame->dts();
else _fps = 1000.0 / (frame->dts() - _tmp);
} else getVideoResolution((uint8_t*)frame->data(), frame->size());
return false;
}
return VideoTrack::inputFrame(frame);
}
void JPEGTrack::getVideoResolution(const uint8_t *buf, int len) {
for (int i = 0; i < len - 8; i++) {
if (buf[i] != 0xff)
continue;
if (buf[i + 1] == 0xC0 /*SOF0*/) {
_height = buf[i + 5] * 256 + buf[i + 6];
_width = buf[i + 7] * 256 + buf[i + 8];
return;
}
}
}
class JPEGSdp : public Sdp {
public:
JPEGSdp(int bitrate) : Sdp(90000, Rtsp::PT_JPEG) {
_printer << "m=video 0 RTP/AVP " << (int)getPayloadType() << "\r\n";
if (bitrate) {
_printer << "b=AS:" << bitrate << "\r\n";
}
}
std::string getSdp() const { return _printer; }
private:
_StrPrinter _printer;
};
Sdp::Ptr JPEGTrack::getSdp(uint8_t) const {
return std::make_shared<JPEGSdp>(getBitRate() / 1024);
}
namespace {
CodecId getCodec() {
return CodecJPEG;
}
Track::Ptr getTrackByCodecId(int sample_rate, int channels, int sample_bit) {
return std::make_shared<JPEGTrack>();
}
Track::Ptr getTrackBySdp(const SdpTrack::Ptr &track) {
return std::make_shared<JPEGTrack>();
}
RtpCodec::Ptr getRtpEncoderByCodecId(uint8_t pt) {
return std::make_shared<JPEGRtpEncoder>();
}
RtpCodec::Ptr getRtpDecoderByCodecId() {
return std::make_shared<JPEGRtpDecoder>();
}
RtmpCodec::Ptr getRtmpEncoderByTrack(const Track::Ptr &track) {
WarnL << "Unsupported jpeg rtmp encoder";
return nullptr;
}
RtmpCodec::Ptr getRtmpDecoderByTrack(const Track::Ptr &track) {
WarnL << "Unsupported jpeg rtmp decoder";
return nullptr;
}
Frame::Ptr getFrameFromPtr(const char *data, size_t bytes, uint64_t dts, uint64_t pts) {
return std::make_shared<JPEGFrame<FrameFromPtr>>(0, CodecJPEG, (char *)data, bytes, dts, pts);
}
} // namespace
CodecPlugin jpeg_plugin = { getCodec,
getTrackByCodecId,
getTrackBySdp,
getRtpEncoderByCodecId,
getRtpDecoderByCodecId,
getRtmpEncoderByTrack,
getRtmpDecoderByTrack,
getFrameFromPtr };
} // namespace mediakit

64
ext-codec/JPEG.h Normal file
View File

@@ -0,0 +1,64 @@
#ifndef ZLMEDIAKIT_JPEG_H
#define ZLMEDIAKIT_JPEG_H
#include "Extension/Frame.h"
#include "Extension/Track.h"
namespace mediakit {
class JPEGTrack : public VideoTrack {
public:
using Ptr = std::shared_ptr<JPEGTrack>;
CodecId getCodecId() const override { return CodecJPEG; }
int getVideoHeight() const override { return _height; }
int getVideoWidth() const override { return _width; }
float getVideoFps() const override { return _fps; }
bool ready() const override { return _fps > 0; }
bool inputFrame(const Frame::Ptr &frame) override;
private:
Sdp::Ptr getSdp(uint8_t payload_type) const override;
Track::Ptr clone() const override { return std::make_shared<JPEGTrack>(*this); }
void getVideoResolution(const uint8_t *buf, int len);
private:
int _width = 0;
int _height = 0;
float _fps = 0;
uint64_t _tmp = 0;
};
class JPEGFrameType {
public:
virtual ~JPEGFrameType() = default;
virtual uint8_t pixType() const = 0;
};
template <typename Parent>
class JPEGFrame : public Parent, public JPEGFrameType {
public:
static constexpr auto kJFIFSize = 20u;
/**
* JPEG/MJPEG帧
* @param pix_type pixel format type; AV_PIX_FMT_YUVJ422P || (AVCOL_RANGE_JPEG && AV_PIX_FMT_YUV422P) : 1; AV_PIX_FMT_YUVJ420P || (AVCOL_RANGE_JPEG && AV_PIX_FMT_YUV420P) : 0
*/
template <typename... ARGS>
JPEGFrame(uint8_t pix_type, ARGS &&...args) : Parent(std::forward<ARGS>(args)...) {
_pix_type = pix_type;
// JFIF头固定20个字节长度
CHECK(this->size() > kJFIFSize);
}
size_t prefixSize() const override { return 0; }
bool keyFrame() const override { return true; }
bool configFrame() const override { return false; }
CodecId getCodecId() const override { return CodecJPEG; }
uint8_t pixType() const override { return _pix_type; }
private:
uint8_t _pix_type;
};
}//namespace mediakit
#endif //ZLMEDIAKIT_JPEG_H

829
ext-codec/JPEGRtp.cpp Normal file
View File

@@ -0,0 +1,829 @@
#include "JPEGRtp.h"
#include "JPEG.h"
using namespace std;
using namespace mediakit;
#define AV_WB24(p, d) \
do { \
((uint8_t *)(p))[2] = (d); \
((uint8_t *)(p))[1] = (d) >> 8; \
((uint8_t *)(p))[0] = (d) >> 16; \
} while (0)
#define AV_WB16(p, d) \
do { \
((uint8_t *)(p))[1] = (d); \
((uint8_t *)(p))[0] = (d) >> 8; \
} while (0)
#define AV_WB8(p, d) do { ((uint8_t*)(p))[0] = (d); } while(0)
/* JPEG marker codes */
enum JpegMarker {
/* start of frame */
SOF0 = 0xc0, /* baseline */
SOF1 = 0xc1, /* extended sequential, huffman */
SOF2 = 0xc2, /* progressive, huffman */
SOF3 = 0xc3, /* lossless, huffman */
SOF5 = 0xc5, /* differential sequential, huffman */
SOF6 = 0xc6, /* differential progressive, huffman */
SOF7 = 0xc7, /* differential lossless, huffman */
JPG = 0xc8, /* reserved for JPEG extension */
SOF9 = 0xc9, /* extended sequential, arithmetic */
SOF10 = 0xca, /* progressive, arithmetic */
SOF11 = 0xcb, /* lossless, arithmetic */
SOF13 = 0xcd, /* differential sequential, arithmetic */
SOF14 = 0xce, /* differential progressive, arithmetic */
SOF15 = 0xcf, /* differential lossless, arithmetic */
DHT = 0xc4, /* define huffman tables */
DAC = 0xcc, /* define arithmetic-coding conditioning */
/* restart with modulo 8 count "m" */
RST0 = 0xd0,
RST1 = 0xd1,
RST2 = 0xd2,
RST3 = 0xd3,
RST4 = 0xd4,
RST5 = 0xd5,
RST6 = 0xd6,
RST7 = 0xd7,
SOI = 0xd8, /* start of image */
EOI = 0xd9, /* end of image */
SOS = 0xda, /* start of scan */
DQT = 0xdb, /* define quantization tables */
DNL = 0xdc, /* define number of lines */
DRI = 0xdd, /* define restart interval */
DHP = 0xde, /* define hierarchical progression */
EXP = 0xdf, /* expand reference components */
APP0 = 0xe0,
APP1 = 0xe1,
APP2 = 0xe2,
APP3 = 0xe3,
APP4 = 0xe4,
APP5 = 0xe5,
APP6 = 0xe6,
APP7 = 0xe7,
APP8 = 0xe8,
APP9 = 0xe9,
APP10 = 0xea,
APP11 = 0xeb,
APP12 = 0xec,
APP13 = 0xed,
APP14 = 0xee,
APP15 = 0xef,
JPG0 = 0xf0,
JPG1 = 0xf1,
JPG2 = 0xf2,
JPG3 = 0xf3,
JPG4 = 0xf4,
JPG5 = 0xf5,
JPG6 = 0xf6,
SOF48 = 0xf7, ///< JPEG-LS
LSE = 0xf8, ///< JPEG-LS extension parameters
JPG9 = 0xf9,
JPG10 = 0xfa,
JPG11 = 0xfb,
JPG12 = 0xfc,
JPG13 = 0xfd,
COM = 0xfe, /* comment */
TEM = 0x01, /* temporary private use for arithmetic coding */
/* 0x02 -> 0xbf reserved */
};
typedef struct PutByteContext {
uint8_t *buffer, *buffer_end, *buffer_start;
int eof;
} PutByteContext;
static void bytestream2_init_writer(PutByteContext *p, uint8_t *buf, int buf_size) {
assert(buf_size >= 0);
p->buffer = buf;
p->buffer_start = buf;
p->buffer_end = buf + buf_size;
p->eof = 0;
}
static inline void bytestream2_put_byte(PutByteContext *p, uint8_t value) {
if (!p->eof && (p->buffer_end - p->buffer >= 1)) {
p->buffer[0] = value;
p->buffer += 1;
} else {
p->eof = 1;
}
}
static inline void bytestream2_put_be16(PutByteContext *p, uint16_t value) {
if (!p->eof && (p->buffer_end - p->buffer >= 2)) {
p->buffer[0] = value >> 8;
p->buffer[1] = value & 0x00FF;
p->buffer += 2;
} else {
p->eof = 1;
}
}
static inline void bytestream2_put_be24(PutByteContext *p, uint16_t value) {
if (!p->eof && (p->buffer_end - p->buffer >= 2)) {
p->buffer[0] = value >> 16;
p->buffer[1] = value >> 8;
p->buffer[2] = value & 0x00FF;
p->buffer += 2;
} else {
p->eof = 1;
}
}
static unsigned int bytestream2_put_buffer(PutByteContext *p, const uint8_t *src, unsigned int size) {
int size2 = 0;
if (p->eof) {
return 0;
}
size2 = MIN(p->buffer_end - p->buffer, size);
if (size2 != (int)size) {
p->eof = 1;
}
memcpy(p->buffer, src, size2);
p->buffer += size2;
return size2;
}
static inline int bytestream2_tell_p(PutByteContext *p) {
return (int) (p->buffer - p->buffer_start);
}
static inline void avio_write(string &str, const void *ptr, size_t size) {
str.append((char *) ptr, size);
}
//////////////////////////////////////////////////////////////////////////////////////////////////
static const uint8_t default_quantizers[128] = {
/* luma table */
16, 11, 12, 14, 12, 10, 16, 14,
13, 14, 18, 17, 16, 19, 24, 40,
26, 24, 22, 22, 24, 49, 35, 37,
29, 40, 58, 51, 61, 60, 57, 51,
56, 55, 64, 72, 92, 78, 64, 68,
87, 69, 55, 56, 80, 109, 81, 87,
95, 98, 103, 104, 103, 62, 77, 113,
121, 112, 100, 120, 92, 101, 103, 99,
/* chroma table */
17, 18, 18, 24, 21, 24, 47, 26,
26, 47, 99, 66, 56, 66, 99, 99,
99, 99, 99, 99, 99, 99, 99, 99,
99, 99, 99, 99, 99, 99, 99, 99,
99, 99, 99, 99, 99, 99, 99, 99,
99, 99, 99, 99, 99, 99, 99, 99,
99, 99, 99, 99, 99, 99, 99, 99,
99, 99, 99, 99, 99, 99, 99, 99
};
/* Set up the standard Huffman tables (cf. JPEG standard section K.3) */
/* IMPORTANT: these are only valid for 8-bit data precision! */
const uint8_t avpriv_mjpeg_bits_dc_luminance[17] =
{ /* 0-base */ 0, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0};
const uint8_t avpriv_mjpeg_val_dc[12] =
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
const uint8_t avpriv_mjpeg_bits_dc_chrominance[17] =
{ /* 0-base */ 0, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0};
const uint8_t avpriv_mjpeg_bits_ac_luminance[17] =
{ /* 0-base */ 0, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7d};
const uint8_t avpriv_mjpeg_val_ac_luminance[] =
{0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12,
0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07,
0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08,
0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0,
0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16,
0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28,
0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39,
0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49,
0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59,
0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69,
0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79,
0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89,
0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98,
0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6,
0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5,
0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4,
0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2,
0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea,
0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8,
0xf9, 0xfa
};
const uint8_t avpriv_mjpeg_bits_ac_chrominance[17] =
{ /* 0-base */ 0, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77};
const uint8_t avpriv_mjpeg_val_ac_chrominance[] =
{0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21,
0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71,
0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91,
0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0,
0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34,
0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26,
0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38,
0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48,
0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68,
0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96,
0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5,
0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4,
0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3,
0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2,
0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda,
0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9,
0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8,
0xf9, 0xfa
};
static int jpeg_create_huffman_table(PutByteContext *p, int table_class,
int table_id, const uint8_t *bits_table,
const uint8_t *value_table) {
int i = 0, n = 0;
bytestream2_put_byte(p, table_class << 4 | table_id);
for (i = 1; i <= 16; i++) {
n += bits_table[i];
bytestream2_put_byte(p, bits_table[i]);
}
for (i = 0; i < n; i++) {
bytestream2_put_byte(p, value_table[i]);
}
return n + 17;
}
static void jpeg_put_marker(PutByteContext *pbc, int code) {
bytestream2_put_byte(pbc, 0xff);
bytestream2_put_byte(pbc, code);
}
static int jpeg_create_header(uint8_t *buf, int size, uint32_t type, uint32_t w,
uint32_t h, const uint8_t *qtable, int nb_qtable,
int dri) {
PutByteContext pbc;
uint8_t *dht_size_ptr;
int dht_size = 0, i = 0;
bytestream2_init_writer(&pbc, buf, size);
/* Convert from blocks to pixels. */
w <<= 3;
h <<= 3;
/* SOI */
jpeg_put_marker(&pbc, SOI);
/* JFIF header */
jpeg_put_marker(&pbc, APP0);
bytestream2_put_be16(&pbc, 16);
bytestream2_put_buffer(&pbc, (const uint8_t *) "JFIF", 5);
bytestream2_put_be16(&pbc, 0x0201);
bytestream2_put_byte(&pbc, 0);
bytestream2_put_be16(&pbc, 1);
bytestream2_put_be16(&pbc, 1);
bytestream2_put_byte(&pbc, 0);
bytestream2_put_byte(&pbc, 0);
if (dri) {
jpeg_put_marker(&pbc, DRI);
bytestream2_put_be16(&pbc, 4);
bytestream2_put_be16(&pbc, dri);
}
/* DQT */
jpeg_put_marker(&pbc, DQT);
bytestream2_put_be16(&pbc, 2 + nb_qtable * (1 + 64));
for (i = 0; i < nb_qtable; i++) {
bytestream2_put_byte(&pbc, i);
/* Each table is an array of 64 values given in zig-zag
* order, identical to the format used in a JFIF DQT
* marker segment. */
bytestream2_put_buffer(&pbc, qtable + 64 * i, 64);
}
/* DHT */
jpeg_put_marker(&pbc, DHT);
dht_size_ptr = pbc.buffer;
bytestream2_put_be16(&pbc, 0);
dht_size = 2;
dht_size += jpeg_create_huffman_table(&pbc, 0, 0, avpriv_mjpeg_bits_dc_luminance,
avpriv_mjpeg_val_dc);
dht_size += jpeg_create_huffman_table(&pbc, 0, 1, avpriv_mjpeg_bits_dc_chrominance,
avpriv_mjpeg_val_dc);
dht_size += jpeg_create_huffman_table(&pbc, 1, 0, avpriv_mjpeg_bits_ac_luminance,
avpriv_mjpeg_val_ac_luminance);
dht_size += jpeg_create_huffman_table(&pbc, 1, 1, avpriv_mjpeg_bits_ac_chrominance,
avpriv_mjpeg_val_ac_chrominance);
AV_WB16(dht_size_ptr, dht_size);
/* SOF0 */
jpeg_put_marker(&pbc, SOF0);
bytestream2_put_be16(&pbc, 17); /* size */
bytestream2_put_byte(&pbc, 8); /* bits per component */
bytestream2_put_be16(&pbc, h);
bytestream2_put_be16(&pbc, w);
bytestream2_put_byte(&pbc, 3); /* number of components */
bytestream2_put_byte(&pbc, 1); /* component number */
bytestream2_put_byte(&pbc, (2 << 4) | (type ? 2 : 1)); /* hsample/vsample */
bytestream2_put_byte(&pbc, 0); /* matrix number */
bytestream2_put_byte(&pbc, 2); /* component number */
bytestream2_put_byte(&pbc, 1 << 4 | 1); /* hsample/vsample */
bytestream2_put_byte(&pbc, nb_qtable == 2 ? 1 : 0); /* matrix number */
bytestream2_put_byte(&pbc, 3); /* component number */
bytestream2_put_byte(&pbc, 1 << 4 | 1); /* hsample/vsample */
bytestream2_put_byte(&pbc, nb_qtable == 2 ? 1 : 0); /* matrix number */
/* SOS */
jpeg_put_marker(&pbc, SOS);
bytestream2_put_be16(&pbc, 12);
bytestream2_put_byte(&pbc, 3);
bytestream2_put_byte(&pbc, 1);
bytestream2_put_byte(&pbc, 0);
bytestream2_put_byte(&pbc, 2);
bytestream2_put_byte(&pbc, 17);
bytestream2_put_byte(&pbc, 3);
bytestream2_put_byte(&pbc, 17);
bytestream2_put_byte(&pbc, 0);
bytestream2_put_byte(&pbc, 63);
bytestream2_put_byte(&pbc, 0);
/* Return the length in bytes of the JPEG header. */
return bytestream2_tell_p(&pbc);
}
static inline int av_clip(int a, int amin, int amax) {
if (a < amin) { return amin; }
else if (a > amax) { return amax; }
else { return a; }
}
static void create_default_qtables(uint8_t *qtables, uint8_t q) {
int factor = q;
int i = 0;
uint16_t S;
factor = av_clip(q, 1, 99);
if (q < 50) {
S = 5000 / factor;
} else {
S = 200 - factor * 2;
}
for (i = 0; i < 128; i++) {
int val = (default_quantizers[i] * S + 50) / 100;
/* Limit the quantizers to 1 <= q <= 255. */
val = av_clip(val, 1, 255);
qtables[i] = val;
}
}
#define AVERROR_INVALIDDATA -1
#define AVERROR_PATCHWELCOME -2
#define AVERROR_EAGAIN -3
#define RTP_FLAG_KEY 0x1 ///< RTP packet contains a keyframe
#define RTP_FLAG_MARKER 0x2 ///< RTP marker bit was set for this packet
#define av_log(ctx, level, ...) PrintD(__VA_ARGS__)
#ifndef AV_RB24
# define AV_RB24(x) \
((((const uint8_t*)(x))[0] << 16) | \
(((const uint8_t*)(x))[1] << 8) | \
((const uint8_t*)(x))[2])
#endif
#define AV_RB8(x) (((const uint8_t*)(x))[0])
#ifndef AV_RB16
# define AV_RB16(x) ((((const uint8_t*)(x))[0] << 8) | (((const uint8_t*)(x))[1] ))
#endif
static int jpeg_parse_packet(void *ctx, PayloadContext *jpeg, uint32_t *timestamp, const uint8_t *buf, int len,
uint16_t seq, int flags, uint8_t *type) {
uint8_t q = 0, width = 0, height = 0;
const uint8_t *qtables = NULL;
uint16_t qtable_len = 0;
uint32_t off = 0;
int ret = 0, dri = 0;
if (len < 8) {
av_log(ctx, AV_LOG_ERROR, "Too short RTP/JPEG packet.\n");
return AVERROR_INVALIDDATA;
}
/* Parse the main JPEG header. */
off = AV_RB24(buf + 1); /* fragment byte offset */
*type = AV_RB8(buf + 4); /* id of jpeg decoder params */
q = AV_RB8(buf + 5); /* quantization factor (or table id) */
width = AV_RB8(buf + 6); /* frame width in 8 pixel blocks */
height = AV_RB8(buf + 7); /* frame height in 8 pixel blocks */
buf += 8;
len -= 8;
if (*type & 0x40) {
if (len < 4) {
av_log(ctx, AV_LOG_ERROR, "Too short RTP/JPEG packet.\n");
return AVERROR_INVALIDDATA;
}
dri = AV_RB16(buf);
buf += 4;
len -= 4;
*type &= ~0x40;
}
if (*type > 1) {
av_log(ctx, AV_LOG_ERROR, "RTP/JPEG type %d", (int) *type);
return AVERROR_PATCHWELCOME;
}
/* Parse the quantization table header. */
if (off == 0) {
/* Start of JPEG data packet. */
uint8_t new_qtables[128];
uint8_t hdr[1024];
if (q > 127) {
uint8_t precision;
if (len < 4) {
av_log(ctx, AV_LOG_ERROR, "Too short RTP/JPEG packet.\n");
return AVERROR_INVALIDDATA;
}
/* The first byte is reserved for future use. */
precision = AV_RB8(buf + 1); /* size of coefficients */
qtable_len = AV_RB16(buf + 2); /* length in bytes */
buf += 4;
len -= 4;
if (precision) {
av_log(ctx, AV_LOG_WARNING, "Only 8-bit precision is supported.\n");
}
if (qtable_len > 0) {
if (len < qtable_len) {
av_log(ctx, AV_LOG_ERROR, "Too short RTP/JPEG packet.\n");
return AVERROR_INVALIDDATA;
}
qtables = buf;
buf += qtable_len;
len -= qtable_len;
if (q < 255) {
if (jpeg->qtables_len[q - 128] &&
(jpeg->qtables_len[q - 128] != qtable_len ||
memcmp(qtables, &jpeg->qtables[q - 128][0], qtable_len))) {
av_log(ctx, AV_LOG_WARNING,
"Quantization tables for q=%d changed\n", q);
} else if (!jpeg->qtables_len[q - 128] && qtable_len <= 128) {
memcpy(&jpeg->qtables[q - 128][0], qtables,
qtable_len);
jpeg->qtables_len[q - 128] = qtable_len;
}
}
} else {
if (q == 255) {
av_log(ctx, AV_LOG_ERROR,
"Invalid RTP/JPEG packet. Quantization tables not found.\n");
return AVERROR_INVALIDDATA;
}
if (!jpeg->qtables_len[q - 128]) {
av_log(ctx, AV_LOG_ERROR,
"No quantization tables known for q=%d yet.\n", q);
return AVERROR_INVALIDDATA;
}
qtables = &jpeg->qtables[q - 128][0];
qtable_len = jpeg->qtables_len[q - 128];
}
} else { /* q <= 127 */
if (q == 0 || q > 99) {
av_log(ctx, AV_LOG_ERROR, "Reserved q value %d\n", q);
return AVERROR_INVALIDDATA;
}
create_default_qtables(new_qtables, q);
qtables = new_qtables;
qtable_len = sizeof(new_qtables);
}
/* Skip the current frame in case of the end packet
* has been lost somewhere. */
jpeg->frame.clear();
jpeg->frame.reserve(1024 + len);
jpeg->timestamp = *timestamp;
/* Generate a frame and scan headers that can be prepended to the
* RTP/JPEG data payload to produce a JPEG compressed image in
* interchange format. */
jpeg->hdr_size = jpeg_create_header(hdr, sizeof(hdr), *type, width,
height, qtables,
qtable_len / 64, dri);
/* Copy JPEG header to frame buffer. */
avio_write(jpeg->frame, hdr, jpeg->hdr_size);
}
if (jpeg->frame.empty()) {
av_log(ctx, AV_LOG_ERROR,
"Received packet without a start chunk; dropping frame.\n");
return AVERROR_EAGAIN;
}
if (jpeg->timestamp != *timestamp) {
/* Skip the current frame if timestamp is incorrect.
* A start packet has been lost somewhere. */
jpeg->frame.clear();
av_log(ctx, AV_LOG_ERROR, "RTP timestamps don't match.\n");
return AVERROR_INVALIDDATA;
}
if (off != jpeg->frame.size() - jpeg->hdr_size) {
av_log(ctx, AV_LOG_ERROR,
"Missing packets; dropping frame.\n");
return AVERROR_EAGAIN;
}
/* Copy data to frame buffer. */
avio_write(jpeg->frame, buf, len);
if (flags & RTP_FLAG_MARKER) {
/* End of JPEG data packet. */
uint8_t buf[2] = {0xff, EOI};
/* Put EOI marker. */
avio_write(jpeg->frame, buf, sizeof(buf));
return 0;
}
return AVERROR_EAGAIN;
}
//----------------------------------------------------------------------------------
#define DEF(type, name, bytes, write) \
static inline void bytestream_put_##name(uint8_t **b, const type value) { \
write(*b, value); \
(*b) += bytes; \
}
DEF(unsigned int, be24, 3, AV_WB24)
DEF(unsigned int, be16, 2, AV_WB16)
DEF(unsigned int, byte, 1, AV_WB8)
static inline void bytestream_put_buffer(uint8_t **b, const uint8_t *src, unsigned int size) {
memcpy(*b, src, size);
(*b) += size;
}
void JPEGRtpEncoder::rtpSendJpeg(const uint8_t *buf, int size, uint64_t pts, uint8_t type)
{
const uint8_t *qtables[4] = { NULL };
int nb_qtables = 0;
uint8_t w, h;
uint8_t *p;
int off = 0; /* fragment offset of the current JPEG frame */
int len;
int i;
int default_huffman_tables = 0;
uint8_t *out = nullptr;
/* preparse the header for getting some info */
for (i = 0; i < size; i++) {
if (buf[i] != 0xff)
continue;
if (buf[i + 1] == DQT) {
int tables, j;
if (buf[i + 4] & 0xF0)
av_log(s1, AV_LOG_WARNING,
"Only 8-bit precision is supported.\n");
/* a quantization table is 64 bytes long */
tables = AV_RB16(&buf[i + 2]) / 65;
if (i + 5 + tables * 65 > size) {
av_log(s1, AV_LOG_ERROR, "Too short JPEG header. Aborted!\n");
return;
}
if (nb_qtables + tables > 4) {
av_log(s1, AV_LOG_ERROR, "Invalid number of quantisation tables\n");
return;
}
for (j = 0; j < tables; j++)
qtables[nb_qtables + j] = buf + i + 5 + j * 65;
nb_qtables += tables;
// 大致忽略DQT/qtable所占字节数，提高搜寻速度
i += tables << 6;
} else if (buf[i + 1] == SOF0) {
if (buf[i + 14] != 17 || buf[i + 17] != 17) {
av_log(s1, AV_LOG_ERROR,
"Only 1x1 chroma blocks are supported. Aborted!\n");
return;
}
h = (buf[i + 5] * 256 + buf[i + 6]) / 8;
w = (buf[i + 7] * 256 + buf[i + 8]) / 8;
// 大致忽略SOF0所占字节数，提高搜寻速度
i += 16;
} else if (buf[i + 1] == DHT) {
int dht_size = AV_RB16(&buf[i + 2]);
default_huffman_tables |= 1 << 4;
i += 3;
dht_size -= 2;
if (i + dht_size >= size)
continue;
while (dht_size > 0)
switch (buf[i + 1]) {
case 0x00:
if ( dht_size >= 29
&& !memcmp(buf + i + 2, avpriv_mjpeg_bits_dc_luminance + 1, 16)
&& !memcmp(buf + i + 18, avpriv_mjpeg_val_dc, 12)) {
default_huffman_tables |= 1;
i += 29;
dht_size -= 29;
} else {
i += dht_size;
dht_size = 0;
}
break;
case 0x01:
if ( dht_size >= 29
&& !memcmp(buf + i + 2, avpriv_mjpeg_bits_dc_chrominance + 1, 16)
&& !memcmp(buf + i + 18, avpriv_mjpeg_val_dc, 12)) {
default_huffman_tables |= 1 << 1;
i += 29;
dht_size -= 29;
} else {
i += dht_size;
dht_size = 0;
}
break;
case 0x10:
if ( dht_size >= 179
&& !memcmp(buf + i + 2, avpriv_mjpeg_bits_ac_luminance + 1, 16)
&& !memcmp(buf + i + 18, avpriv_mjpeg_val_ac_luminance, 162)) {
default_huffman_tables |= 1 << 2;
i += 179;
dht_size -= 179;
} else {
i += dht_size;
dht_size = 0;
}
break;
case 0x11:
if ( dht_size >= 179
&& !memcmp(buf + i + 2, avpriv_mjpeg_bits_ac_chrominance + 1, 16)
&& !memcmp(buf + i + 18, avpriv_mjpeg_val_ac_chrominance, 162)) {
default_huffman_tables |= 1 << 3;
i += 179;
dht_size -= 179;
} else {
i += dht_size;
dht_size = 0;
}
break;
default:
i += dht_size;
dht_size = 0;
continue;
}
} else if (buf[i + 1] == SOS) {
/* SOS is last marker in the header */
i += AV_RB16(&buf[i + 2]) + 2;
if (i > size) {
av_log(s1, AV_LOG_ERROR,
"Insufficient data. Aborted!\n");
return;
}
break;
}
}
if (default_huffman_tables && default_huffman_tables != 31) {
av_log(s1, AV_LOG_ERROR,
"RFC 2435 requires standard Huffman tables for jpeg\n");
return;
}
if (nb_qtables && nb_qtables != 2)
av_log(s1, AV_LOG_WARNING,
"RFC 2435 suggests two quantization tables, %d provided\n",
nb_qtables);
/* skip JPEG header */
buf += i;
size -= i;
for (i = size - 2; i >= 0; i--) {
if (buf[i] == 0xff && buf[i + 1] == EOI) {
/* Remove the EOI marker */
size = i;
break;
}
}
while (size > 0) {
int hdr_size = 8;
if (off == 0 && nb_qtables)
hdr_size += 4 + 64 * nb_qtables;
/* payload max in one packet */
len = MIN(size, (int)getRtpInfo().getMaxSize() - hdr_size);
/* marker bit is last packet in frame */
auto rtp_packet = getRtpInfo().makeRtp(TrackVideo, nullptr, len + hdr_size, size == len, pts);
p = rtp_packet->getPayload();
/* set main header */
bytestream_put_byte(&p, 0);
bytestream_put_be24(&p, off);
bytestream_put_byte(&p, type);
bytestream_put_byte(&p, 255);
bytestream_put_byte(&p, w);
bytestream_put_byte(&p, h);
if (off == 0 && nb_qtables) {
/* set quantization tables header */
bytestream_put_byte(&p, 0);
bytestream_put_byte(&p, 0);
bytestream_put_be16(&p, 64 * nb_qtables);
for (i = 0; i < nb_qtables; i++)
bytestream_put_buffer(&p, qtables[i], 64);
}
/* copy payload data */
memcpy(p, buf, len);
// output rtp packet
RtpCodec::inputRtp(std::move(rtp_packet), false);
buf += len;
size -= len;
off += len;
}
free(out);
}
////////////////////////////////////////////////////////////
JPEGRtpDecoder::JPEGRtpDecoder() {
memset(&_ctx.timestamp, 0, sizeof(_ctx) - offsetof(decltype(_ctx), timestamp));
}
using JPEGFrameImp = JPEGFrame<FrameFromBuffer<FrameFromPtr> >;
bool JPEGRtpDecoder::inputRtp(const RtpPacket::Ptr &rtp, bool) {
auto payload = rtp->getPayload();
auto size = rtp->getPayloadSize();
auto stamp = rtp->getStamp();
auto seq = rtp->getSeq();
auto marker = rtp->getHeader()->mark;
if (size <= 0) {
//无实际负载
return false;
}
uint8_t type;
if (0 == jpeg_parse_packet(nullptr, &_ctx, &stamp, payload, size, seq, marker ? RTP_FLAG_MARKER : 0, &type)) {
auto buffer = std::make_shared<toolkit::BufferString>(std::move(_ctx.frame));
auto frame = std::make_shared<JPEGFrameImp>(type, std::move(buffer), stamp / 90, 0);
_ctx.frame.clear();
RtpCodec::inputFrame(std::move(frame));
}
return false;
}
////////////////////////////////////////////////////////////////////////
bool JPEGRtpEncoder::inputFrame(const Frame::Ptr &frame) {
// JFIF头固定20个字节长度
auto ptr = (uint8_t *)frame->data() + frame->prefixSize() + JPEGFrameImp::kJFIFSize;
auto len = frame->size() - frame->prefixSize() - JPEGFrameImp::kJFIFSize;
auto pts = frame->pts();
auto type = 1;
auto jpeg = dynamic_pointer_cast<JPEGFrameType>(frame);
if (jpeg) {
type = jpeg->pixType();
}
rtpSendJpeg(ptr, len, pts, type);
return len > 0;
}

50
ext-codec/JPEGRtp.h Normal file
View File

@@ -0,0 +1,50 @@
#ifndef ZLMEDIAKIT_JPEGRTP_H
#define ZLMEDIAKIT_JPEGRTP_H
#include "Rtsp/RtpCodec.h"
#include "Extension/Frame.h"
namespace mediakit {
/**
* RTP/JPEG specific private data.
*/
struct PayloadContext {
std::string frame; ///< current frame buffer
uint32_t timestamp; ///< current frame timestamp
int hdr_size; ///< size of the current frame header
uint8_t qtables[128][128];
uint8_t qtables_len[128];
};
/**
* 通用 rtp解码类
*/
class JPEGRtpDecoder : public RtpCodec {
public:
typedef std::shared_ptr <JPEGRtpDecoder> Ptr;
JPEGRtpDecoder();
/**
* 输入rtp并解码
* @param rtp rtp数据包
* @param key_pos 此参数内部强制转换为false,请忽略之
*/
bool inputRtp(const RtpPacket::Ptr &rtp, bool key_pos = false) override;
private:
struct PayloadContext _ctx;
};
class JPEGRtpEncoder : public RtpCodec {
public:
using Ptr = std::shared_ptr<JPEGRtpEncoder>;
bool inputFrame(const Frame::Ptr &frame) override;
private:
void rtpSendJpeg(const uint8_t *buf, int size, uint64_t pts, uint8_t type);
};
}//namespace mediakit
#endif //ZLMEDIAKIT_JPEGRTP_H

109
ext-codec/L16.cpp Normal file
View File

@@ -0,0 +1,109 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "L16.h"
#include "Extension/Factory.h"
#include "Extension/CommonRtp.h"
#include "Extension/CommonRtmp.h"
using namespace std;
using namespace toolkit;
namespace mediakit {
/**
* L16类型SDP
*/
class L16Sdp : public Sdp {
public:
/**
* L16采样位数固定为16位
* @param payload_type rtp payload type
* @param channels 通道数
* @param sample_rate 音频采样率
* @param bitrate 比特率
*/
L16Sdp(int payload_type, int sample_rate, int channels, int bitrate) : Sdp(sample_rate, payload_type) {
_printer << "m=audio 0 RTP/AVP " << payload_type << "\r\n";
if (bitrate) {
_printer << "b=AS:" << bitrate << "\r\n";
}
_printer << "a=rtpmap:" << payload_type << " " << getCodecName(CodecL16) << "/" << sample_rate << "/" << channels << "\r\n";
}
string getSdp() const override { return _printer; }
private:
_StrPrinter _printer;
};
Sdp::Ptr L16Track::getSdp(uint8_t payload_type) const {
WarnL << "Enter L16Track::getSdp function";
if (!ready()) {
WarnL << getCodecName() << " Track未准备好";
return nullptr;
}
return std::make_shared<L16Sdp>(payload_type, getAudioSampleRate(), getAudioChannel(), getBitRate() / 1024);
}
Track::Ptr L16Track::clone() const {
return std::make_shared<L16Track>(*this);
}
namespace {
CodecId getCodec() {
return CodecL16;
}
Track::Ptr getTrackByCodecId(int sample_rate, int channels, int sample_bit) {
return std::make_shared<L16Track>(sample_rate, channels);
}
Track::Ptr getTrackBySdp(const SdpTrack::Ptr &track) {
return std::make_shared<L16Track>(track->_samplerate, track->_channel);
}
RtpCodec::Ptr getRtpEncoderByCodecId(uint8_t pt) {
return std::make_shared<CommonRtpEncoder>();
}
RtpCodec::Ptr getRtpDecoderByCodecId() {
return std::make_shared<CommonRtpDecoder>(CodecL16);
}
RtmpCodec::Ptr getRtmpEncoderByTrack(const Track::Ptr &track) {
WarnL << "Unsupported L16 rtmp encoder";
return nullptr;
}
RtmpCodec::Ptr getRtmpDecoderByTrack(const Track::Ptr &track) {
WarnL << "Unsupported L16 rtmp decoder";
return nullptr;
}
Frame::Ptr getFrameFromPtr(const char *data, size_t bytes, uint64_t dts, uint64_t pts) {
return std::make_shared<FrameFromPtr>(CodecL16, (char *)data, bytes, dts, pts);
}
} // namespace
CodecPlugin l16_plugin = { getCodec,
getTrackByCodecId,
getTrackBySdp,
getRtpEncoderByCodecId,
getRtpDecoderByCodecId,
getRtmpEncoderByTrack,
getRtmpDecoderByTrack,
getFrameFromPtr };
}//namespace mediakit

33
ext-codec/L16.h Normal file
View File

@@ -0,0 +1,33 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_L16_H
#define ZLMEDIAKIT_L16_H
#include "Extension/Frame.h"
#include "Extension/Track.h"
namespace mediakit {
/**
* L16音频通道
*/
class L16Track : public AudioTrackImp{
public:
using Ptr = std::shared_ptr<L16Track>;
L16Track(int sample_rate, int channels) : AudioTrackImp(CodecL16,sample_rate,channels,16){}
private:
Sdp::Ptr getSdp(uint8_t payload_type) const override;
Track::Ptr clone() const override;
};
}//namespace mediakit
#endif //ZLMEDIAKIT_L16_H

102
ext-codec/Opus.cpp Normal file
View File

@@ -0,0 +1,102 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#include "Opus.h"
#include "Extension/Factory.h"
#include "Extension/CommonRtp.h"
#include "Extension/CommonRtmp.h"
using namespace std;
using namespace toolkit;
namespace mediakit {
/**
* Opus类型SDP
*/
class OpusSdp : public Sdp {
public:
/**
* 构造opus sdp
* @param payload_type rtp payload type
* @param sample_rate 音频采样率
* @param channels 通道数
* @param bitrate 比特率
*/
OpusSdp(int payload_type, int sample_rate, int channels, int bitrate) : Sdp(sample_rate, payload_type) {
_printer << "m=audio 0 RTP/AVP " << payload_type << "\r\n";
if (bitrate) {
_printer << "b=AS:" << bitrate << "\r\n";
}
_printer << "a=rtpmap:" << payload_type << " " << getCodecName(CodecOpus) << "/" << sample_rate << "/" << channels << "\r\n";
}
string getSdp() const override {
return _printer;
}
private:
_StrPrinter _printer;
};
Sdp::Ptr OpusTrack::getSdp(uint8_t payload_type) const {
if (!ready()) {
WarnL << getCodecName() << " Track未准备好";
return nullptr;
}
return std::make_shared<OpusSdp>(payload_type, getAudioSampleRate(), getAudioChannel(), getBitRate() / 1024);
}
namespace {
CodecId getCodec() {
return CodecOpus;
}
Track::Ptr getTrackByCodecId(int sample_rate, int channels, int sample_bit) {
return std::make_shared<OpusTrack>();
}
Track::Ptr getTrackBySdp(const SdpTrack::Ptr &track) {
return std::make_shared<OpusTrack>();
}
RtpCodec::Ptr getRtpEncoderByCodecId(uint8_t pt) {
return std::make_shared<CommonRtpEncoder>();
}
RtpCodec::Ptr getRtpDecoderByCodecId() {
return std::make_shared<CommonRtpDecoder>(CodecOpus);
}
RtmpCodec::Ptr getRtmpEncoderByTrack(const Track::Ptr &track) {
return std::make_shared<CommonRtmpEncoder>(track);
}
RtmpCodec::Ptr getRtmpDecoderByTrack(const Track::Ptr &track) {
return std::make_shared<CommonRtmpDecoder>(track);
}
Frame::Ptr getFrameFromPtr(const char *data, size_t bytes, uint64_t dts, uint64_t pts) {
return std::make_shared<FrameFromPtr>(CodecOpus, (char *)data, bytes, dts, pts);
}
} // namespace
CodecPlugin opus_plugin = { getCodec,
getTrackByCodecId,
getTrackBySdp,
getRtpEncoderByCodecId,
getRtpDecoderByCodecId,
getRtmpEncoderByTrack,
getRtmpDecoderByTrack,
getFrameFromPtr };
}//namespace mediakit

37
ext-codec/Opus.h Normal file
View File

@@ -0,0 +1,37 @@
/*
* Copyright (c) 2016-present The ZLMediaKit project authors. All Rights Reserved.
*
* This file is part of ZLMediaKit(https://github.com/ZLMediaKit/ZLMediaKit).
*
* Use of this source code is governed by MIT-like license that can be found in the
* LICENSE file in the root of the source tree. All contributing project authors
* may be found in the AUTHORS file in the root of the source tree.
*/
#ifndef ZLMEDIAKIT_OPUS_H
#define ZLMEDIAKIT_OPUS_H
#include "Extension/Frame.h"
#include "Extension/Track.h"
namespace mediakit {
/**
* Opus帧音频通道
*/
class OpusTrack : public AudioTrackImp{
public:
using Ptr = std::shared_ptr<OpusTrack>;
OpusTrack() : AudioTrackImp(CodecOpus,48000,2,16){}
private:
//克隆该Track
Track::Ptr clone() const override {
return std::make_shared<OpusTrack>(*this);
}
//生成sdp
Sdp::Ptr getSdp(uint8_t payload_type) const override ;
};
}//namespace mediakit
#endif //ZLMEDIAKIT_OPUS_H

2288
ext-codec/SPSParser.c Normal file
View File

File diff suppressed because it is too large Load Diff

465
ext-codec/SPSParser.h Normal file
View File

@@ -0,0 +1,465 @@
#ifndef _SPSPARSER_H_
#define _SPSPARSER_H_
#if defined (__cplusplus)
extern "C" {
#endif
#define QP_MAX_NUM (51 + 6*6) // The maximum supported qp
#define HEVC_MAX_SHORT_TERM_RPS_COUNT 64
#define T_PROFILE_HEVC_MAIN 1
#define T_PROFILE_HEVC_MAIN_10 2
#define T_PROFILE_HEVC_MAIN_STILL_PICTURE 3
#define T_PROFILE_HEVC_REXT 4
/**
* Chromaticity coordinates of the source primaries.
*/
enum T_AVColorPrimaries {
AVCOL_PRI_RESERVED0 = 0,
AVCOL_PRI_BT709 = 1, ///< also ITU-R BT1361 / IEC 61966-2-4 / SMPTE RP177 Annex B
AVCOL_PRI_UNSPECIFIED = 2,
AVCOL_PRI_RESERVED = 3,
AVCOL_PRI_BT470M = 4, ///< also FCC Title 47 Code of Federal Regulations 73.682 (a)(20)
AVCOL_PRI_BT470BG = 5, ///< also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM
AVCOL_PRI_SMPTE170M = 6, ///< also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC
AVCOL_PRI_SMPTE240M = 7, ///< functionally identical to above
AVCOL_PRI_FILM = 8, ///< colour filters using Illuminant C
AVCOL_PRI_BT2020 = 9, ///< ITU-R BT2020
AVCOL_PRI_NB, ///< Not part of ABI
};
/**
* Color Transfer Characteristic.
*/
enum T_AVColorTransferCharacteristic {
AVCOL_TRC_RESERVED0 = 0,
AVCOL_TRC_BT709 = 1, ///< also ITU-R BT1361
AVCOL_TRC_UNSPECIFIED = 2,
AVCOL_TRC_RESERVED = 3,
AVCOL_TRC_GAMMA22 = 4, ///< also ITU-R BT470M / ITU-R BT1700 625 PAL & SECAM
AVCOL_TRC_GAMMA28 = 5, ///< also ITU-R BT470BG
AVCOL_TRC_SMPTE170M = 6, ///< also ITU-R BT601-6 525 or 625 / ITU-R BT1358 525 or 625 / ITU-R BT1700 NTSC
AVCOL_TRC_SMPTE240M = 7,
AVCOL_TRC_LINEAR = 8, ///< "Linear transfer characteristics"
AVCOL_TRC_LOG = 9, ///< "Logarithmic transfer characteristic (100:1 range)"
AVCOL_TRC_LOG_SQRT = 10, ///< "Logarithmic transfer characteristic (100 * Sqrt(10) : 1 range)"
AVCOL_TRC_IEC61966_2_4 = 11, ///< IEC 61966-2-4
AVCOL_TRC_BT1361_ECG = 12, ///< ITU-R BT1361 Extended Colour Gamut
AVCOL_TRC_IEC61966_2_1 = 13, ///< IEC 61966-2-1 (sRGB or sYCC)
AVCOL_TRC_BT2020_10 = 14, ///< ITU-R BT2020 for 10 bit system
AVCOL_TRC_BT2020_12 = 15, ///< ITU-R BT2020 for 12 bit system
AVCOL_TRC_NB, ///< Not part of ABI
};
/**
* YUV tColorspace type.
*/
enum T_AVColorSpace {
AVCOL_SPC_RGB = 0, ///< order of coefficients is actually GBR, also IEC 61966-2-1 (sRGB)
AVCOL_SPC_BT709 = 1, ///< also ITU-R BT1361 / IEC 61966-2-4 xvYCC709 / SMPTE RP177 Annex B
AVCOL_SPC_UNSPECIFIED = 2,
AVCOL_SPC_RESERVED = 3,
AVCOL_SPC_FCC = 4, ///< FCC Title 47 Code of Federal Regulations 73.682 (a)(20)
AVCOL_SPC_BT470BG = 5, ///< also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM / IEC 61966-2-4 xvYCC601
AVCOL_SPC_SMPTE170M = 6, ///< also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC / functionally identical to above
AVCOL_SPC_SMPTE240M = 7,
AVCOL_SPC_YCOCG = 8, ///< Used by Dirac / VC-2 and H.264 FRext, see ITU-T SG16
AVCOL_SPC_BT2020_NCL = 9, ///< ITU-R BT2020 non-constant luminance system
AVCOL_SPC_BT2020_CL = 10, ///< ITU-R BT2020 constant luminance system
AVCOL_SPC_NB, ///< Not part of ABI
};
enum {
// 7.4.3.1: vps_max_layers_minus1 is in [0, 62].
HEVC_MAX_LAYERS = 63,
// 7.4.3.1: vps_max_sub_layers_minus1 is in [0, 6].
HEVC_MAX_SUB_LAYERS = 7,
// 7.4.3.1: vps_num_layer_sets_minus1 is in [0, 1023].
HEVC_MAX_LAYER_SETS = 1024,
// 7.4.2.1: vps_video_parameter_set_id is u(4).
HEVC_MAX_VPS_COUNT = 16,
// 7.4.3.2.1: sps_seq_parameter_set_id is in [0, 15].
HEVC_MAX_SPS_COUNT = 16,
// 7.4.3.3.1: pps_pic_parameter_set_id is in [0, 63].
HEVC_MAX_PPS_COUNT = 64,
// A.4.2: MaxDpbSize is bounded above by 16.
HEVC_MAX_DPB_SIZE = 16,
// 7.4.3.1: vps_max_dec_pic_buffering_minus1[i] is in [0, MaxDpbSize - 1].
HEVC_MAX_REFS = HEVC_MAX_DPB_SIZE,
// 7.4.3.2.1: num_short_term_ref_pic_sets is in [0, 64].
HEVC_MAX_SHORT_TERM_REF_PIC_SETS = 64,
// 7.4.3.2.1: num_long_term_ref_pics_sps is in [0, 32].
HEVC_MAX_LONG_TERM_REF_PICS = 32,
// A.3: all profiles require that CtbLog2SizeY is in [4, 6].
HEVC_MIN_LOG2_CTB_SIZE = 4,
HEVC_MAX_LOG2_CTB_SIZE = 6,
// E.3.2: cpb_cnt_minus1[i] is in [0, 31].
HEVC_MAX_CPB_CNT = 32,
// A.4.1: in table A.6 the highest level allows a MaxLumaPs of 35 651 584.
HEVC_MAX_LUMA_PS = 35651584,
// A.4.1: pic_width_in_luma_samples and pic_height_in_luma_samples are
// constrained to be not greater than sqrt(MaxLumaPs * 8). Hence height/
// width are bounded above by sqrt(8 * 35651584) = 16888.2 samples.
HEVC_MAX_WIDTH = 16888,
HEVC_MAX_HEIGHT = 16888,
// A.4.1: table A.6 allows at most 22 tile rows for any level.
HEVC_MAX_TILE_ROWS = 22,
// A.4.1: table A.6 allows at most 20 tile columns for any level.
HEVC_MAX_TILE_COLUMNS = 20,
// 7.4.7.1: in the worst case (tiles_enabled_flag and
// entropy_coding_sync_enabled_flag are both set), entry points can be
// placed at the beginning of every Ctb row in every tile, giving an
// upper bound of (num_tile_columns_minus1 + 1) * PicHeightInCtbsY - 1.
// Only a stream with very high resolution and perverse parameters could
// get near that, though, so set a lower limit here with the maximum
// possible value for 4K video (at most 135 16x16 Ctb rows).
HEVC_MAX_ENTRY_POINT_OFFSETS = HEVC_MAX_TILE_COLUMNS * 135,
};
/**
* rational number numerator/denominator
*/
typedef struct T_AVRational{
int num; ///< numerator
int den; ///< denominator
} T_AVRational;
/***
* Sequence parameter set
* ¿É²Î¿¼H264±ê×¼µÚ7½ÚºÍ¸½Â¼D E
*/
#define Extended_SAR 255
/**
* Sequence parameter set
*/
typedef struct T_SPS {
unsigned int uiSpsId;
int iProfileIdc;
int iLevelIdc;
int iChromaFormatIdc;
int iTransformBypass; ///< qpprime_y_zero_transform_bypass_flag
int iLog2MaxFrameNum; ///< log2_max_frame_num_minus4 + 4
int iPocType; ///< pic_order_cnt_type
int iLog2MaxPocLsb; ///< log2_max_pic_order_cnt_lsb_minus4
int iDeltaPicOrderAlwaysZeroFlag;
int iOffsetForNonRefPic;
int iOffsetForTopToBottomField;
int iPocCycleLength; ///< num_ref_frames_in_pic_order_cnt_cycle
int iRefFrameCount; ///< num_ref_frames
int iGapsInFrameNumAllowedFlag;
int iMbWidth; ///< pic_width_in_mbs_minus1 + 1
int iMbHeight; ///< pic_height_in_map_units_minus1 + 1
int iFrameMbsOnlyFlag;
int iMbAff; ///< mb_adaptive_frame_field_flag
int iDirect8x8InferenceFlag;
int iCrop; ///< frame_cropping_flag
/* those 4 are already in luma samples */
unsigned int uiCropLeft; ///< frame_cropping_rect_left_offset
unsigned int uiCropRight; ///< frame_cropping_rect_right_offset
unsigned int uiCropTop; ///< frame_cropping_rect_top_offset
unsigned int uiCropBottom; ///< frame_cropping_rect_bottom_offset
int iVuiParametersPresentFlag;
T_AVRational tSar;
int iVideoSignalTypePresentFlag;
int iFullRange;
int iColourDescriptionPresentFlag;
enum T_AVColorPrimaries tColorPrimaries;
enum T_AVColorTransferCharacteristic tColorTrc;
enum T_AVColorSpace tColorspace;
int iTimingInfoPresentFlag;
uint32_t u32NumUnitsInTick;
uint32_t u32TimeScale;
int iFixedFrameRateFlag;
short asOffsetForRefFrame[256]; // FIXME dyn aloc?
int iBitstreamRestrictionFlag;
int iNumReorderFrames;
int iScalingMatrixPresent;
uint8_t aau8ScalingMatrix4[6][16];
uint8_t aau8ScalingMatrix8[6][64];
int iNalHrdParametersPresentFlag;
int iVclHrdParametersPresentFlag;
int iPicStructPresentFlag;
int iTimeOffsetLength;
int iCpbCnt; ///< See H.264 E.1.2
int iInitialCpbRemovalDelayLength; ///< initial_cpb_removal_delay_length_minus1 + 1
int iCpbRemovalDelayLength; ///< cpb_removal_delay_length_minus1 + 1
int iDpbOutputDelayLength; ///< dpb_output_delay_length_minus1 + 1
int iBitDepthLuma; ///< bit_depth_luma_minus8 + 8
int iBitDepthChroma; ///< bit_depth_chroma_minus8 + 8
int iResidualColorTransformFlag; ///< residual_colour_transform_flag
int iConstraintSetFlags; ///< constraint_set[0-3]_flag
int iNew; ///< flag to keep track if the decoder context needs re-init due to changed SPS
} T_SPS;
/**
* Picture parameter set
*/
typedef struct T_PPS {
unsigned int uiSpsId;
int iCabac; ///< entropy_coding_mode_flag
int iPicOrderPresent; ///< pic_order_present_flag
int iSliceGroupCount; ///< num_slice_groups_minus1 + 1
int iMbSliceGroupMapType;
unsigned int auiRefCount[2]; ///< num_ref_idx_l0/1_active_minus1 + 1
int iWeightedPred; ///< weighted_pred_flag
int iWeightedBipredIdc;
int iInitQp; ///< pic_init_qp_minus26 + 26
int iInitQs; ///< pic_init_qs_minus26 + 26
int aiChromaQpIndexOffset[2];
int iDeblockingFilterParametersPresent; ///< deblocking_filter_parameters_present_flag
int iConstrainedIntraPred; ///< constrained_intra_pred_flag
int iRedundantPicCntPresent; ///< redundant_pic_cnt_present_flag
int iTransform8x8Mode; ///< transform_8x8_mode_flag
uint8_t aau8ScalingMatrix4[6][16];
uint8_t aau8ScalingMatrix8[6][64];
uint8_t u8ChromaQpTable[2][QP_MAX_NUM+1]; ///< pre-scaled (with aiChromaQpIndexOffset) version of qp_table
int iChromaQpDiff;
} T_PPS;
typedef struct T_HEVCWindow {
unsigned int uiLeftOffset;
unsigned int uiRightOffset;
unsigned int uiTopOffset;
unsigned int uiBottomOffset;
} T_HEVCWindow;
typedef struct T_VUI {
T_AVRational tSar;
int iOverscanInfoPresentFlag;
int iOverscanAppropriateFlag;
int iVideoSignalTypePresentFlag;
int iVideoFormat;
int iVideoFullRangeFlag;
int iColourDescriptionPresentFlag;
uint8_t u8ColourPrimaries;
uint8_t u8TransferCharacteristic;
uint8_t u8MatrixCoeffs;
int iChromaLocInfoPresentFlag;
int iChromaSampleLocTypeTopField;
int iChromaSampleLocTypeBottomField;
int iNeutraChromaIndicationFlag;
int iFieldSeqFlag;
int iFrameFieldInfoPresentFlag;
int iDefaultDisplayWindowFlag;
T_HEVCWindow tDefDispWin;
int iVuiTimingInfoPresentFlag;
uint32_t u32VuiNumUnitsInTick;
uint32_t u32VuiTimeScale;
int iVuiPocProportionalToTimingFlag;
int iVuiNumTicksPocDiffOneMinus1;
int iVuiHrdParametersPresentFlag;
int iBitstreamRestrictionFlag;
int iTilesFixedStructureFlag;
int iMotionVectorsOverPicBoundariesFlag;
int iRestrictedRefPicListsFlag;
int iMinSpatialSegmentationIdc;
int iMaxBytesPerPicDenom;
int iMaxBitsPerMinCuDenom;
int iLog2MaxMvLengthHorizontal;
int iLog2MaxMvLengthVertical;
} T_VUI;
typedef struct T_PTLCommon {
uint8_t u8ProfileSpace;
uint8_t u8TierFlag;
uint8_t u8ProfileIdc;
uint8_t au8ProfileCompatibilityFlag[32];
uint8_t u8LevelIdc;
uint8_t u8ProgressiveSourceFlag;
uint8_t u8InterlacedSourceFlag;
uint8_t u8NonPackedConstraintFlag;
uint8_t u8FrameOnlyConstraintFlag;
} T_PTLCommon;
typedef struct T_PTL {
T_PTLCommon tGeneralPtl;
T_PTLCommon atSubLayerPtl[HEVC_MAX_SUB_LAYERS];
uint8_t au8SubLayerProfilePresentFlag[HEVC_MAX_SUB_LAYERS];
uint8_t au8SubLayerLevelPresentFlag[HEVC_MAX_SUB_LAYERS];
} T_PTL;
typedef struct T_ScalingList {
/* This is a little wasteful, since sizeID 0 only needs 8 coeffs,
* and size ID 3 only has 2 arrays, not 6. */
uint8_t aaau8Sl[4][6][64];
uint8_t aau8SlDc[2][6];
} T_ScalingList;
typedef struct T_ShortTermRPS {
unsigned int uiNumNegativePics;
int iNumDeltaPocs;
int iRpsIdxNumDeltaPocs;
int32_t au32DeltaPoc[32];
uint8_t au8Used[32];
} T_ShortTermRPS;
typedef struct T_HEVCVPS {
uint8_t u8VpsTemporalIdNestingFlag;
int iVpsMaxLayers;
int iVpsMaxSubLayers; ///< vps_max_temporal_layers_minus1 + 1
T_PTL tPtl;
int iVpsSubLayerOrderingInfoPresentFlag;
unsigned int uiVpsMaxDecPicBuffering[HEVC_MAX_SUB_LAYERS];
unsigned int auiVpsNumReorderPics[HEVC_MAX_SUB_LAYERS];
unsigned int auiVpsMaxLatencyIncrease[HEVC_MAX_SUB_LAYERS];
int iVpsMaxLayerId;
int iVpsNumLayerSets; ///< vps_num_layer_sets_minus1 + 1
uint8_t u8VpsTimingInfoPresentFlag;
uint32_t u32VpsNumUnitsInTick;
uint32_t u32VpsTimeScale;
uint8_t u8VpsPocProportionalToTimingFlag;
int iVpsNumTicksPocDiffOne; ///< vps_num_ticks_poc_diff_one_minus1 + 1
int iVpsNumHrdParameters;
} T_HEVCVPS;
typedef struct T_HEVCSPS {
unsigned int uiVpsId;
int iChromaFormatIdc;
uint8_t u8SeparateColourPlaneFlag;
///< output (i.e. cropped) values
int iIutputWidth, iOutputHeight;
T_HEVCWindow tOutputWindow;
T_HEVCWindow tPicConfWin;
int iBitDepth;
int iBitDepthChroma;
int iPixelShift;
unsigned int uiLog2MaxPocLsb;
int iPcmEnabledFlag;
int iMaxSubLayers;
struct {
int iMaxDecPicBuffering;
int iNumReorderPics;
int iMaxLatencyIncrease;
} stTemporalLayer[HEVC_MAX_SUB_LAYERS];
uint8_t u8temporalIdNestingFlag;
T_VUI tVui;
T_PTL tPtl;
uint8_t u8ScalingListEnableFlag;
T_ScalingList tScalingList;
unsigned int uiNbStRps;
T_ShortTermRPS atStRps[HEVC_MAX_SHORT_TERM_RPS_COUNT];
uint8_t u8AmpEnabledFlag;
uint8_t u8SaoEnabled;
uint8_t u8LongTermRefPicsPresentFlag;
uint16_t au16LtRefPicPocLsbSps[32];
uint8_t au8UsedByCurrPicLtSpsFlag[32];
uint8_t u8NumLongTermRefPicsSps;
struct {
uint8_t u8BitDepth;
uint8_t u8BitDepthChroma;
unsigned int uiLog2MinPcmCbSize;
unsigned int uiLog2MaxPcmCbSize;
uint8_t u8LoopFilterDisableFlag;
} pcm;
uint8_t u8SpsTemporalMvpEnabledFlag;
uint8_t u8SpsStrongIntraMmoothingEnableFlag;
unsigned int uiLog2MinCbSize;
unsigned int uiLog2DiffMaxMinCodingBlockSize;
unsigned int uiLog2MinTbSize;
unsigned int uiLog2MaxTrafoSize;
unsigned int uiLog2CtbSize;
unsigned int uiLog2MinPuSize;
int iMaxTransformHierarchyDepthInter;
int iMaxTransformHierarchyDepthIntra;
int iTransformSkipRotationEnabledFlag;
int iTransformSkipContextEnabledFlag;
int iImplicitRdpcmEnabledFlag;
int iExplicitRdpcmEnabledFlag;
int iIntraSmoothingDisabledFlag;
int iHighPrecisionOffsetsEnabledFlag;
int iPersistentRiceAdaptationEnabledFlag;
///< coded frame dimension in various units
int iWidth;
int iHeight;
int iCtbWidth;
int iCtbHeight;
int iCtbSize;
int iMinCbWidth;
int iMinCbHeight;
int iMinTbWidth;
int iMinTbHeight;
int iMinPuWidth;
int iMinPuHeight;
int iTbMask;
int aiHshift[3];
int aiVshift[3];
int iQpBdOffset;
int iVuiPresent;
}T_HEVCSPS;
typedef struct T_GetBitContext{
uint8_t *pu8Buf; // buf
int iBufSize; // buf size
int iBitPos; // bit position
int iTotalBit; // bit number
int iCurBitPos; // current bit position
}T_GetBitContext;
int h264DecSeqParameterSet(void *pvBuf, T_SPS *ptSps);
int h265DecSeqParameterSet( void *pvBufSrc, T_HEVCSPS *ptSps );
int h265DecVideoParameterSet( void *pvBufSrc, T_HEVCVPS *ptVps );
void h264GetWidthHeight(T_SPS *ptSps, int *piWidth, int *piHeight);
void h265GetWidthHeight(T_HEVCSPS *ptSps, int *piWidth, int *piHeight);
void h264GeFramerate(T_SPS *ptSps, float *pfFramerate);
void h265GeFramerate(T_HEVCVPS *ptVps, T_HEVCSPS *ptSps,float *pfFramerate);
#if defined (__cplusplus)
}
#endif
#endif //_SPS_PPS_H_