diff --git a/CMakeLists.txt b/CMakeLists.txt index 696f60407..be3386b51 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,6 +82,7 @@ set(LIBDATACHANNEL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/h264rtpdepacketizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/nalunit.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/h265rtppacketizer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/h265rtpdepacketizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/h265nalunit.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/av1rtppacketizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/rtcpnackresponder.cpp @@ -120,6 +121,7 @@ set(LIBDATACHANNEL_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h264rtpdepacketizer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/nalunit.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h265rtppacketizer.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h265rtpdepacketizer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h265nalunit.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/av1rtppacketizer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/rtcpnackresponder.hpp diff --git a/include/rtc/h265rtpdepacketizer.hpp b/include/rtc/h265rtpdepacketizer.hpp new file mode 100644 index 000000000..cb239b958 --- /dev/null +++ b/include/rtc/h265rtpdepacketizer.hpp @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2020 Staz Modrzynski + * Copyright (c) 2020-2024 Paul-Louis Ageneau + * Copyright (c) 2024 Robert Edmonds + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#ifndef RTC_H265_RTP_DEPACKETIZER_H +#define RTC_H265_RTP_DEPACKETIZER_H + +#if RTC_ENABLE_MEDIA + +#include "common.hpp" +#include "h265nalunit.hpp" +#include "mediahandler.hpp" +#include "message.hpp" +#include "rtp.hpp" + +#include + +namespace rtc { + +/// RTP depacketization for H265 +class RTC_CPP_EXPORT H265RtpDepacketizer : public MediaHandler { +public: + using Separator = NalUnit::Separator; + + H265RtpDepacketizer(Separator separator = Separator::LongStartSequence); + virtual ~H265RtpDepacketizer() = default; + + void incoming(message_vector &messages, const message_callback &send) override; + +private: + std::vector mRtpBuffer; + const NalUnit::Separator separator; + + void addSeparator(binary& accessUnit); + message_vector buildFrames(message_vector::iterator firstPkt, message_vector::iterator lastPkt, + uint8_t payloadType, uint32_t timestamp); +}; + +} // namespace rtc + +#endif // RTC_ENABLE_MEDIA + +#endif // RTC_H265_RTP_DEPACKETIZER_H diff --git a/include/rtc/rtc.hpp b/include/rtc/rtc.hpp index 8a2b33010..74a9c9e8b 100644 --- a/include/rtc/rtc.hpp +++ b/include/rtc/rtc.hpp @@ -32,6 +32,7 @@ #include "h264rtppacketizer.hpp" #include "h264rtpdepacketizer.hpp" #include "h265rtppacketizer.hpp" +#include "h265rtpdepacketizer.hpp" #include "mediahandler.hpp" #include "plihandler.hpp" #include "rembhandler.hpp" diff --git a/src/h265nalunit.cpp b/src/h265nalunit.cpp index 5fda10545..6f6b7e12a 100644 --- a/src/h265nalunit.cpp +++ b/src/h265nalunit.cpp @@ -34,7 +34,7 @@ H265NalUnitFragment::fragmentsFrom(shared_ptr nalu, uint16_t maxFra auto fragments_count = ceil(double(nalu->size()) / maxFragmentSize); maxFragmentSize = uint16_t(int(ceil(nalu->size() / fragments_count))); - // 3 bytes for FU indicator and FU header + // 3 bytes for NALU header and FU header maxFragmentSize -= (H265_NAL_HEADER_SIZE + H265_FU_HEADER_SIZE); auto f = nalu->forbiddenBit(); uint8_t nuhLayerId = nalu->nuhLayerId() & 0x3F; // 6 bits diff --git a/src/h265rtpdepacketizer.cpp b/src/h265rtpdepacketizer.cpp new file mode 100644 index 000000000..ae9f4ba7d --- /dev/null +++ b/src/h265rtpdepacketizer.cpp @@ -0,0 +1,193 @@ +/** + * Copyright (c) 2023-2024 Paul-Louis Ageneau + * Copyright (c) 2024 Robert Edmonds + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#if RTC_ENABLE_MEDIA + +#include "h265rtpdepacketizer.hpp" +#include "h265nalunit.hpp" + +#include "impl/internals.hpp" + +#include + +namespace rtc { + +const binary naluLongStartCode = {byte{0}, byte{0}, byte{0}, byte{1}}; +const binary naluShortStartCode = {byte{0}, byte{0}, byte{1}}; + +const uint8_t naluTypeAP = 48; +const uint8_t naluTypeFU = 49; + +H265RtpDepacketizer::H265RtpDepacketizer(Separator separator) : separator(separator) { + switch (separator) { + case Separator::StartSequence: [[fallthrough]]; + case Separator::LongStartSequence: [[fallthrough]]; + case Separator::ShortStartSequence: + break; + case Separator::Length: [[fallthrough]]; + default: + throw std::invalid_argument("Invalid separator"); + } +} + +void H265RtpDepacketizer::addSeparator(binary& accessUnit) +{ + switch (separator) { + case Separator::StartSequence: [[fallthrough]]; + case Separator::LongStartSequence: + accessUnit.insert(accessUnit.end(), + naluLongStartCode.begin(), + naluLongStartCode.end()); + break; + case Separator::ShortStartSequence: + accessUnit.insert(accessUnit.end(), + naluShortStartCode.begin(), + naluShortStartCode.end()); + break; + case Separator::Length: [[fallthrough]]; + default: + throw std::invalid_argument("Invalid separator"); + } +} + +message_vector H265RtpDepacketizer::buildFrames(message_vector::iterator begin, + message_vector::iterator end, + uint8_t payloadType, uint32_t timestamp) { + message_vector out = {}; + auto accessUnit = binary{}; + auto frameInfo = std::make_shared(payloadType, timestamp); + + for (auto it = begin; it != end; ++it) { + auto pkt = it->get(); + auto pktParsed = reinterpret_cast(pkt->data()); + auto rtpHeaderSize = pktParsed->getSize() + pktParsed->getExtensionHeaderSize(); + auto rtpPaddingSize = 0; + + if (pktParsed->padding()) { + rtpPaddingSize = std::to_integer(pkt->at(pkt->size() - 1)); + } + + if (pkt->size() == rtpHeaderSize + rtpPaddingSize) { + PLOG_VERBOSE << "H.265 RTP packet has empty payload"; + continue; + } + + auto nalUnitHeader = + H265NalUnitHeader{std::to_integer(pkt->at(rtpHeaderSize)), + std::to_integer(pkt->at(rtpHeaderSize + 1))}; + + if (nalUnitHeader.unitType() == naluTypeFU) { + auto nalUnitFragmentHeader = H265NalUnitFragmentHeader{ + std::to_integer(pkt->at(rtpHeaderSize + sizeof(H265NalUnitHeader)))}; + + // RFC 7798: "When set to 1, the S bit indicates the start of a fragmented + // NAL unit, i.e., the first byte of the FU payload is also the first byte of + // the payload of the fragmented NAL unit. When the FU payload is not the start + // of the fragmented NAL unit payload, the S bit MUST be set to 0." + if (nalUnitFragmentHeader.isStart() || accessUnit.empty()) { + addSeparator(accessUnit); + nalUnitHeader.setUnitType(nalUnitFragmentHeader.unitType()); + accessUnit.emplace_back(byte(nalUnitHeader._first)); + accessUnit.emplace_back(byte(nalUnitHeader._second)); + } + + accessUnit.insert(accessUnit.end(), + pkt->begin() + rtpHeaderSize + sizeof(H265NalUnitHeader) + + sizeof(H265NalUnitFragmentHeader), + pkt->end()); + } else if (nalUnitHeader.unitType() == naluTypeAP) { + auto currOffset = rtpHeaderSize + sizeof(H265NalUnitHeader); + + while (currOffset + sizeof(uint16_t) < pkt->size()) { + auto naluSize = std::to_integer(pkt->at(currOffset)) << 8 | + std::to_integer(pkt->at(currOffset + 1)); + + currOffset += sizeof(uint16_t); + + if (pkt->size() < currOffset + naluSize) { + throw std::runtime_error("H265 AP declared size is larger than buffer"); + } + + addSeparator(accessUnit); + accessUnit.insert(accessUnit.end(), pkt->begin() + currOffset, + pkt->begin() + currOffset + naluSize); + + currOffset += naluSize; + } + } else if (nalUnitHeader.unitType() < naluTypeAP) { + // "NAL units with NAL unit type values in the range of 0 to 47, inclusive, may be + // passed to the decoder." + addSeparator(accessUnit); + accessUnit.insert(accessUnit.end(), pkt->begin() + rtpHeaderSize, pkt->end()); + } else { + // "NAL-unit-like structures with NAL unit type values in the range of 48 to 63, + // inclusive, MUST NOT be passed to the decoder." + } + } + + if (!accessUnit.empty()) { + out.emplace_back(make_message(accessUnit.begin(), accessUnit.end(), Message::Binary, 0, + nullptr, frameInfo)); + } + + return out; +} + +void H265RtpDepacketizer::incoming(message_vector &messages, const message_callback &) { + messages.erase(std::remove_if(messages.begin(), messages.end(), + [&](message_ptr message) { + if (message->type == Message::Control) { + return false; + } + + if (message->size() < sizeof(RtpHeader)) { + PLOG_VERBOSE << "RTP packet is too small, size=" + << message->size(); + return true; + } + + mRtpBuffer.push_back(std::move(message)); + return true; + }), + messages.end()); + + while (mRtpBuffer.size() != 0) { + uint8_t payload_type = 0; + uint32_t current_timestamp = 0; + size_t packets_in_timestamp = 0; + + for (const auto &pkt : mRtpBuffer) { + auto p = reinterpret_cast(pkt->data()); + + if (current_timestamp == 0) { + current_timestamp = p->timestamp(); + payload_type = p->payloadType(); // should all be the same for data of the same codec + } else if (current_timestamp != p->timestamp()) { + break; + } + + packets_in_timestamp++; + } + + if (packets_in_timestamp == mRtpBuffer.size()) { + break; + } + + auto begin = mRtpBuffer.begin(); + auto end = mRtpBuffer.begin() + (packets_in_timestamp - 1); + + auto frames = buildFrames(begin, end + 1, payload_type, current_timestamp); + messages.insert(messages.end(), frames.begin(), frames.end()); + mRtpBuffer.erase(mRtpBuffer.begin(), mRtpBuffer.begin() + packets_in_timestamp); + } +} + +} // namespace rtc + +#endif // RTC_ENABLE_MEDIA