Add UDP video protocol + client/server screen sharing support

.github/actions/install-dependencies/install_macos_static_x86_64.sh

@@ -7,7 +7,7 @@ source "$( dirname "$0" )/common.sh"
 
 verify_required_env_variables_set
 
-brew install coreutils aria2 gnu-tar xz
+brew install coreutils aria2 gnu-tar xz ffmpeg
 
 make_build_env_available "tar.xz"
 

.github/actions/install-dependencies/install_ubuntu_shared_x86_64.sh

@@ -35,7 +35,10 @@ sudo apt -y install \
 	zsync \
 	appstream \
 	libpoco-dev \
-	libsqlite3-dev
+	libsqlite3-dev \
+	libavcodec-dev \
+	libswscale-dev \
+	libavutil-dev
 
 # The package was initially called libqt6svg6-dev.
 # Choose correct name based on the Ubuntu version along with some other version-specific setup

.github/actions/install-dependencies/install_ubuntu_static_x86_64.sh

@@ -14,6 +14,7 @@ sudo apt -y install \
 	`# Still required for qtbase vcpkg package` \
 	'^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev libegl1-mesa-dev \
 	`# TODO: can we get rid of these by replacing with vcpkg packages?` \
+	libavcodec-dev libswscale-dev libavutil-dev \
 	libsm-dev \
 	libspeechd-dev \
 	libavahi-compat-libdnssd-dev \

.github/actions/install-dependencies/install_windows_static_x86_64.sh

@@ -24,6 +24,9 @@ rm -rf "g15_sdk"
 aria2c "https://github.com/oleg-shilo/wixsharp/releases/download/v1.19.0.0/WixSharp.1.19.0.0.7z" --out "WixSharp.7z"
 extract_with_progress "WixSharp.7z" "C:/WixSharp"
 
+aria2c "https://github.com/GyanD/codexffmpeg/releases/download/8.1/ffmpeg-8.1-full_build-shared.zip" --out "ffmpeg.zip"
+extract_with_progress "ffmpeg.zip" "${GITHUB_WORKSPACE}/3rdparty/ffmpeg"
+
 git clone "https://github.com/nathan818fr/vcvars-bash.git" "C:/vcvars-bash"
 
 

src/Mumble.proto

@@ -245,6 +245,8 @@ message UserState {
 	repeated uint32 listening_channel_remove = 22;
 	// A list of volume adjustments the user has applied to listeners
 	repeated VolumeAdjustment listening_volume_adjustment = 23;
+	// True if the user is currently sharing their screen.
+	optional bool screen_sharing = 24;
 }
 
 // Relays information on the bans. The client may send the BanList message to

src/MumbleProtocol.cpp

@@ -625,6 +625,13 @@ namespace Protocol {
 					return decodeAudio_protobuf(data.subspan(1, data.size() - 1));
 				case UDPMessageType::Ping:
 					return decodePing_protobuf(data.subspan(1, data.size() - 1));
+				case UDPMessageType::Video:
+					if (restrictToPing) {
+						// Not a ping
+						return false;
+					}
+
+					return decodeVideo_protobuf(data.subspan(1, data.size() - 1));
 			}
 
 			// Unknown package type
@@ -638,6 +645,12 @@ namespace Protocol {
 
 	template< Role role > UDPMessageType UDPDecoder< role >::getMessageType() const { return m_messageType; }
 
+	template< Role role > VideoData UDPDecoder< role >::getVideoData() const {
+		assert(m_messageType == UDPMessageType::Video);
+
+		return m_videoData;
+	}
+
 	template< Role role > AudioData UDPDecoder< role >::getAudioData() const {
 		assert(m_messageType == UDPMessageType::Audio);
 
@@ -848,6 +861,54 @@ namespace Protocol {
 		return true;
 	}
 
+	template< Role role > bool UDPDecoder< role >::decodeVideo_protobuf(const std::span< const byte > data) {
+		m_messageType = UDPMessageType::Video;
+		m_videoData   = {};
+
+		if (!m_videoMessage.ParseFromArray(data.data(), static_cast< int >(data.size()))) {
+			// Invalid format
+			return false;
+		}
+
+		// Sender
+		m_videoData.senderSession = m_videoMessage.sender_session();
+
+		// Codec (string, unlike Audio enum)
+		m_videoData.codec = m_videoMessage.codec();
+
+		// Resolution
+		m_videoData.width  = m_videoMessage.width();
+		m_videoData.height = m_videoMessage.height();
+
+		// Frame / fragmentation info
+		m_videoData.frameNumber   = m_videoMessage.frame_number();
+		m_videoData.fragmentIndex = m_videoMessage.fragment_index();
+		m_videoData.fragmentCount = m_videoMessage.fragment_count();
+		if (m_videoData.fragmentIndex >= m_videoData.fragmentCount) {
+			// Sending more fragments than the fargment count isn't supported.
+			return false;
+		}
+
+		if (m_videoData.width == 0 || m_videoData.height == 0) {
+			// Sending a video with no pixels isn't supported
+			return false;
+		}
+
+		// Payload
+		if (m_videoMessage.video_data().empty()) {
+			// Video packets without payload are invalid
+			return false;
+		}
+
+		std::string &videoPayload = *m_videoMessage.mutable_video_data();
+		m_videoData.payload = std::span< byte >(reinterpret_cast< byte * >(&videoPayload[0]), videoPayload.size());
+
+		// Keyframe flag
+		m_videoData.isKeyFrame = m_videoMessage.is_keyframe();
+
+		return true;
+	}
+
 	template< Role role > bool UDPDecoder< role >::decodeAudio_protobuf(const std::span< const byte > data) {
 		m_messageType = UDPMessageType::Audio;
 		m_audioData   = {};

src/MumbleProtocol.h

@@ -57,7 +57,8 @@
  */
 #define MUMBLE_ALL_UDP_MESSAGES          \
 	PROCESS_MUMBLE_UDP_MESSAGE(Audio, 0) \
-	PROCESS_MUMBLE_UDP_MESSAGE(Ping, 1)
+	PROCESS_MUMBLE_UDP_MESSAGE(Ping, 1)  \
+	PROCESS_MUMBLE_UDP_MESSAGE(Video, 2)
 
 namespace Mumble {
 namespace Protocol {
@@ -149,6 +150,19 @@ namespace Protocol {
 		friend bool operator!=(const AudioData &lhs, const AudioData &rhs);
 	};
 
+	/// Carries all fields from a decoded MumbleUDP::Video fragment.
+	struct VideoData {
+		std::uint32_t senderSession  = 0;
+		MumbleUDP::Video_Codec codec = MumbleUDP::Video_Codec_H264;
+		std::uint32_t width          = 0;
+		std::uint32_t height         = 0;
+		std::uint64_t frameNumber    = 0;
+		std::uint32_t fragmentIndex  = 0;
+		std::uint32_t fragmentCount  = 0;
+		std::span< const byte > payload;
+		bool isKeyFrame = false;
+	};
+
 	struct PingData {
 		std::uint64_t timestamp            = 0;
 		bool requestAdditionalInformation  = false;
@@ -265,20 +279,24 @@ namespace Protocol {
 		UDPMessageType getMessageType() const;
 
 		AudioData getAudioData() const;
+		VideoData getVideoData() const;
 		PingData getPingData() const;
 
 	protected:
 		std::vector< byte > m_byteBuffer;
 		UDPMessageType m_messageType;
 		AudioData m_audioData = {};
+		VideoData m_videoData = {};
 		PingData m_pingData   = {};
 		MumbleUDP::Ping m_pingMessage;
 		MumbleUDP::Audio m_audioMessage;
+		MumbleUDP::Video m_videoMessage;
 
 		bool decodePing_legacy(const std::span< const byte > data);
 		bool decodePing_protobuf(const std::span< const byte > data);
 		bool decodeAudio_legacy(const std::span< const byte > data, AudioCodec codec);
 		bool decodeAudio_protobuf(const std::span< const byte > data);
+		bool decodeVideo_protobuf(const std::span< const byte > data);
 	};
 
 } // namespace Protocol

src/MumbleUDP.proto

@@ -53,6 +53,46 @@ message Audio {
 	bool is_terminator = 16;
 }
 
+/**
+ * Sent by a client to stream an encoded video frame fragment to other users in the same channel.
+ * The server relays each fragment to all other users in the sender's channel.
+ * Large frames are split into multiple fragments; receivers reassemble them using
+ * frame_number + fragment_index / fragment_count.
+ */
+message Video {
+	// The session of the client that sent this fragment.
+	// When the client sends this to the server the field is ignored; the server always
+	// overwrites it with the authenticated sender's session before forwarding.
+	uint32 sender_session = 1;
+
+	// Codec identifier. Currently only "H264" is defined.
+	enum Codec {
+		H264 = 0;
+	}
+	Codec codec = 2;
+
+	// Width of the complete video frame in pixels.
+	uint32 width = 3;
+
+	// Height of the complete video frame in pixels.
+	uint32 height = 4;
+
+	// Monotonically increasing frame counter.
+	uint64 frame_number = 5;
+
+	// Zero-based index of this fragment within the current frame.
+	uint32 fragment_index = 6;
+
+	// Total number of fragments that make up the current frame.
+	uint32 fragment_count = 7;
+
+	// Encoded video bytes for this fragment (H.264 Annex-B NAL unit stream).
+	bytes video_data = 8;
+
+	// True if this frame is a key frame (IDR). Only meaningful on fragment_index 0.
+	bool is_keyframe = 9;
+}
+
 /**
  * Ping message for checking UDP connectivity (and roundtrip ping) and potentially obtaining further server
  * details (e.g. version).

src/User.cpp

@@ -13,6 +13,7 @@ User::User() {
 	bSelfMute = bSelfDeaf = false;
 	bPrioritySpeaker      = false;
 	bRecording            = false;
+	bScreenSharing        = false;
 	bSuppress             = false;
 	cChannel              = 0;
 }

src/User.h

@@ -30,6 +30,7 @@ class User {
 	bool bSelfMute, bSelfDeaf;
 	bool bPrioritySpeaker;
 	bool bRecording;
+	bool bScreenSharing;
 	Channel *cChannel;
 	QByteArray qbaTexture;
 	QByteArray qbaTextureHash;