Merge pull request #1265 from paullouisageneau/sendframe

Add sendFrame() and refactor packetization

Author: paullouisageneau

examples/streamer/main.cpp

@@ -20,6 +20,8 @@
 #include "helpers.hpp"
 #include "ArgParser.hpp"
 
+#include <chrono>
+
 using namespace rtc;
 using namespace std;
 using namespace std::chrono_literals;
@@ -208,7 +210,7 @@ shared_ptr<ClientTrackData> addVideo(const shared_ptr<PeerConnection> pc, const
     video.addSSRC(ssrc, cname, msid, cname);
     auto track = pc->addTrack(video);
     // create RTP configuration
-    auto rtpConfig = make_shared<RtpPacketizationConfig>(ssrc, cname, payloadType, H264RtpPacketizer::defaultClockRate);
+    auto rtpConfig = make_shared<RtpPacketizationConfig>(ssrc, cname, payloadType, H264RtpPacketizer::ClockRate);
     // create packetizer
     auto packetizer = make_shared<H264RtpPacketizer>(NalUnit::Separator::Length, rtpConfig);
     // add RTCP SR handler
@@ -351,26 +353,11 @@ shared_ptr<Stream> createStream(const string h264Samples, const unsigned fps, co
             for (auto clientTrack: tracks) {
                 auto client = clientTrack.id;
                 auto trackData = clientTrack.trackData;
-                auto rtpConfig = trackData->sender->rtpConfig;
-
-                // sample time is in us, we need to convert it to seconds
-                auto elapsedSeconds = double(sampleTime) / (1000 * 1000);
-                // get elapsed time in clock rate
-                uint32_t elapsedTimestamp = rtpConfig->secondsToTimestamp(elapsedSeconds);
-                // set new timestamp
-                rtpConfig->timestamp = rtpConfig->startTimestamp + elapsedTimestamp;
-
-                // get elapsed time in clock rate from last RTCP sender report
-                auto reportElapsedTimestamp = rtpConfig->timestamp - trackData->sender->lastReportedTimestamp();
-                // check if last report was at least 1 second ago
-                if (rtpConfig->timestampToSeconds(reportElapsedTimestamp) > 1) {
-                    trackData->sender->setNeedsToReport();
-                }
 
                 cout << "Sending " << streamType << " sample with size: " << to_string(sample.size()) << " to " << client << endl;
                 try {
                     // send sample
-                    trackData->track->send(sample);
+                    trackData->track->sendFrame(sample, std::chrono::duration<double, std::micro>(sampleTime));
                 } catch (const std::exception &e) {
                     cerr << "Unable to send "<< streamType << " packet: " << e.what() << endl;
                 }

include/rtc/av1rtppacketizer.hpp

@@ -20,8 +20,8 @@ namespace rtc {
 // RTP packetization of AV1 payload
 class RTC_CPP_EXPORT AV1RtpPacketizer final : public RtpPacketizer {
 public:
-	// Default clock rate for AV1 in RTP
-	inline static const uint32_t defaultClockRate = 90 * 1000;
+	inline static const uint32_t ClockRate = VideoClockRate;
+	[[deprecated("Use ClockRate")]] inline static const uint32_t defaultClockRate = ClockRate;
 
 	// Define how OBUs are seperated in a AV1 Sample
 	enum class Packetization {
@@ -33,17 +33,18 @@ class RTC_CPP_EXPORT AV1RtpPacketizer final : public RtpPacketizer {
 	// @note RTP configuration is used in packetization process which may change some configuration
 	// properties such as sequence number.
 	AV1RtpPacketizer(Packetization packetization, shared_ptr<RtpPacketizationConfig> rtpConfig,
-	                 uint16_t maxFragmentSize = NalUnits::defaultMaximumFragmentSize);
-
-	void outgoing(message_vector &messages, const message_callback &send) override;
+	                 size_t maxFragmentSize = DefaultMaxFragmentSize);
 
 private:
-	shared_ptr<NalUnits> splitMessage(binary_ptr message);
-	std::vector<shared_ptr<binary>> packetizeObu(binary_ptr message, uint16_t maxFragmentSize);
+	static std::vector<binary> extractTemporalUnitObus(const binary &data);
+
+	std::vector<binary> fragment(binary data) override;
+	std::vector<binary> fragmentObu(const binary &data);
+
+	const Packetization mPacketization;
+	const size_t mMaxFragmentSize;
 
-	const uint16_t maxFragmentSize;
-	const Packetization packetization;
-	std::shared_ptr<binary> sequenceHeader;
+	std::unique_ptr<binary> mSequenceHeader;
 };
 
 // For backward compatibility, do not use

include/rtc/common.hpp

@@ -67,7 +67,6 @@ using std::variant;
 using std::weak_ptr;
 
 using binary = std::vector<byte>;
-using binary_ptr = shared_ptr<binary>;
 using message_variant = variant<binary, string>;
 
 using std::int16_t;

include/rtc/frameinfo.hpp

@@ -11,12 +11,20 @@
 
 #include "common.hpp"
 
+#include <chrono>
+
 namespace rtc {
 
 struct RTC_CPP_EXPORT FrameInfo {
-	FrameInfo(uint8_t payloadType, uint32_t timestamp) : payloadType(payloadType), timestamp(timestamp){};
-	uint8_t payloadType; // Indicates codec of the frame
-	uint32_t timestamp = 0; // RTP Timestamp
+	FrameInfo(uint32_t timestamp) : timestamp(timestamp) {};
+	template<typename Period = std::ratio<1>> FrameInfo(std::chrono::duration<double, Period> timestamp) : timestampSeconds(timestamp) {};
+
+	[[deprecated]] FrameInfo(uint8_t payloadType, uint32_t timestamp) : timestamp(timestamp), payloadType(payloadType) {};
+
+	uint32_t timestamp = 0;
+	uint8_t payloadType = 0;
+
+	optional<std::chrono::duration<double>> timestampSeconds;
 };
 
 } // namespace rtc

include/rtc/h264rtpdepacketizer.hpp

@@ -27,6 +27,8 @@ class RTC_CPP_EXPORT H264RtpDepacketizer : public MediaHandler {
 public:
 	using Separator = NalUnit::Separator;
 
+	inline static const uint32_t ClockRate = 90 * 1000;
+
 	H264RtpDepacketizer(Separator separator = Separator::LongStartSequence);
 	virtual ~H264RtpDepacketizer() = default;
 

include/rtc/h264rtppacketizer.hpp

@@ -22,8 +22,8 @@ class RTC_CPP_EXPORT H264RtpPacketizer final : public RtpPacketizer {
 public:
 	using Separator = NalUnit::Separator;
 
-	/// Default clock rate for H264 in RTP
-	inline static const uint32_t defaultClockRate = 90 * 1000;
+	inline static const uint32_t ClockRate = VideoClockRate;
+	[[deprecated("Use ClockRate")]] inline static const uint32_t defaultClockRate = ClockRate;
 
 	/// Constructs h264 payload packetizer with given RTP configuration.
 	/// @note RTP configuration is used in packetization process which may change some configuration
@@ -32,20 +32,19 @@ class RTC_CPP_EXPORT H264RtpPacketizer final : public RtpPacketizer {
 	/// @param rtpConfig RTP configuration
 	/// @param maxFragmentSize maximum size of one NALU fragment
 	H264RtpPacketizer(Separator separator, shared_ptr<RtpPacketizationConfig> rtpConfig,
-	                  uint16_t maxFragmentSize = NalUnits::defaultMaximumFragmentSize);
+	                  size_t maxFragmentSize = DefaultMaxFragmentSize);
 
 	// For backward compatibility, do not use
 	[[deprecated]] H264RtpPacketizer(
 	    shared_ptr<RtpPacketizationConfig> rtpConfig,
-	    uint16_t maxFragmentSize = NalUnits::defaultMaximumFragmentSize);
-
-	void outgoing(message_vector &messages, const message_callback &send) override;
+	    size_t maxFragmentSize = DefaultMaxFragmentSize);
 
 private:
-	shared_ptr<NalUnits> splitMessage(binary_ptr message);
+	std::vector<binary> fragment(binary data) override;
+	std::vector<NalUnit> splitFrame(const binary &frame);
 
-	const uint16_t maxFragmentSize;
-	const Separator separator;
+	const Separator mSeparator;
+	const size_t mMaxFragmentSize;
 };
 
 // For backward compatibility, do not use

include/rtc/h265nalunit.hpp

@@ -15,6 +15,7 @@
 #include "nalunit.hpp"
 
 #include <cassert>
+#include <vector>
 
 namespace rtc {
 
@@ -72,8 +73,13 @@ struct RTC_CPP_EXPORT H265NalUnitFragmentHeader {
 
 #pragma pack(pop)
 
-/// Nal unit
+struct H265NalUnitFragment;
+
+/// NAL unit
 struct RTC_CPP_EXPORT H265NalUnit : NalUnit {
+	static std::vector<binary> GenerateFragments(const std::vector<H265NalUnit> &nalus,
+	                                             size_t maxFragmentSize);
+
 	H265NalUnit(const H265NalUnit &unit) = default;
 	H265NalUnit(size_t size, bool includingHeader = true)
 	    : NalUnit(size, includingHeader, NalUnit::Type::H265) {}
@@ -104,6 +110,8 @@ struct RTC_CPP_EXPORT H265NalUnit : NalUnit {
 		insert(end(), payload.begin(), payload.end());
 	}
 
+	std::vector<H265NalUnitFragment> generateFragments(size_t maxFragmentSize) const;
+
 protected:
 	const H265NalUnitHeader *header() const {
 		assert(size() >= H265_NAL_HEADER_SIZE);
@@ -116,9 +124,9 @@ struct RTC_CPP_EXPORT H265NalUnit : NalUnit {
 	}
 };
 
-/// Nal unit fragment A
+/// NAL unit fragment
 struct RTC_CPP_EXPORT H265NalUnitFragment : H265NalUnit {
-	static std::vector<shared_ptr<H265NalUnitFragment>> fragmentsFrom(shared_ptr<H265NalUnit> nalu,
+	[[deprecated]] static std::vector<shared_ptr<H265NalUnitFragment>> fragmentsFrom(shared_ptr<H265NalUnit> nalu,
 	                                                                  uint16_t maxFragmentSize);
 
 	enum class FragmentType { Start, Middle, End };
@@ -171,7 +179,7 @@ struct RTC_CPP_EXPORT H265NalUnitFragment : H265NalUnit {
 	}
 };
 
-class RTC_CPP_EXPORT H265NalUnits : public std::vector<shared_ptr<H265NalUnit>> {
+class [[deprecated]] RTC_CPP_EXPORT H265NalUnits : public std::vector<shared_ptr<H265NalUnit>> {
 public:
 	static const uint16_t defaultMaximumFragmentSize =
 	    uint16_t(RTC_DEFAULT_MTU - 12 - 8 - 40); // SRTP/UDP/IPv6

include/rtc/h265rtpdepacketizer.hpp

@@ -28,6 +28,8 @@ class RTC_CPP_EXPORT H265RtpDepacketizer : public MediaHandler {
 public:
 	using Separator = NalUnit::Separator;
 
+	inline static const uint32_t ClockRate = 90 * 1000;
+
 	H265RtpDepacketizer(Separator separator = Separator::LongStartSequence);
 	virtual ~H265RtpDepacketizer() = default;
 

include/rtc/h265rtppacketizer.hpp

@@ -21,8 +21,8 @@ class RTC_CPP_EXPORT H265RtpPacketizer final : public RtpPacketizer {
 public:
 	using Separator = NalUnit::Separator;
 
-	// Default clock rate for H265 in RTP
-	inline static const uint32_t defaultClockRate = 90 * 1000;
+	inline static const uint32_t ClockRate = VideoClockRate;
+	[[deprecated("Use ClockRate")]] inline static const uint32_t defaultClockRate = ClockRate;
 
 	// Constructs h265 payload packetizer with given RTP configuration.
 	// @note RTP configuration is used in packetization process which may change some configuration
@@ -31,19 +31,18 @@ class RTC_CPP_EXPORT H265RtpPacketizer final : public RtpPacketizer {
 	// @param rtpConfig  RTP configuration
 	// @param maxFragmentSize maximum size of one NALU fragment
 	H265RtpPacketizer(Separator separator, shared_ptr<RtpPacketizationConfig> rtpConfig,
-	                  uint16_t maxFragmentSize = H265NalUnits::defaultMaximumFragmentSize);
+	                  size_t maxFragmentSize = DefaultMaxFragmentSize);
 
-	// for backward compatibility
+	// For backward compatibility, do not use
 	[[deprecated]] H265RtpPacketizer(shared_ptr<RtpPacketizationConfig> rtpConfig,
-	                  uint16_t maxFragmentSize = H265NalUnits::defaultMaximumFragmentSize);
-
-	void outgoing(message_vector &messages, const message_callback &send) override;
+	                  size_t maxFragmentSize = DefaultMaxFragmentSize);
 
 private:
-	shared_ptr<H265NalUnits> splitMessage(binary_ptr message);
+	std::vector<binary> fragment(binary data) override;
+	std::vector<H265NalUnit> splitFrame(const binary &frame);
 
-	const uint16_t maxFragmentSize;
-	const NalUnit::Separator separator;
+	const NalUnit::Separator mSeparator;
+	const size_t mMaxFragmentSize;
 };
 
 // For backward compatibility, do not use

include/rtc/message.hpp

@@ -46,11 +46,26 @@ inline size_t message_size_func(const message_ptr &m) {
 
 template <typename Iterator>
 message_ptr make_message(Iterator begin, Iterator end, Message::Type type = Message::Binary,
-                         unsigned int stream = 0, shared_ptr<Reliability> reliability = nullptr,
-                         shared_ptr<FrameInfo> frameInfo = nullptr) {
+                         unsigned int stream = 0, shared_ptr<Reliability> reliability = nullptr) {
 	auto message = std::make_shared<Message>(begin, end, type);
 	message->stream = stream;
 	message->reliability = reliability;
+	return message;
+}
+
+template <typename Iterator>
+message_ptr make_message(Iterator begin, Iterator end, shared_ptr<FrameInfo> frameInfo) {
+	auto message = std::make_shared<Message>(begin, end);
+	message->frameInfo = frameInfo;
+	return message;
+}
+
+// For backward compatibiity, do not use
+template <typename Iterator>
+[[deprecated]] message_ptr make_message(Iterator begin, Iterator end, Message::Type type,
+                         unsigned int stream, shared_ptr<FrameInfo> frameInfo) {
+	auto message = std::make_shared<Message>(begin, end, type);
+	message->stream = stream;
 	message->frameInfo = frameInfo;
 	return message;
 }
@@ -61,8 +76,9 @@ RTC_CPP_EXPORT message_ptr make_message(size_t size, Message::Type type = Messag
 
 RTC_CPP_EXPORT message_ptr make_message(binary &&data, Message::Type type = Message::Binary,
                                         unsigned int stream = 0,
-                                        shared_ptr<Reliability> reliability = nullptr,
-                                        shared_ptr<FrameInfo> frameInfo = nullptr);
+                                        shared_ptr<Reliability> reliability = nullptr);
+
+RTC_CPP_EXPORT message_ptr make_message(binary &&data, shared_ptr<FrameInfo> frameInfo);
 
 RTC_CPP_EXPORT message_ptr make_message(size_t size, message_ptr orig);