diff --git a/.github/workflows/flutter-build.yml b/.github/workflows/flutter-build.yml index 1c4520522..cac2d770d 100644 --- a/.github/workflows/flutter-build.yml +++ b/.github/workflows/flutter-build.yml @@ -874,7 +874,7 @@ jobs: - name: Test cursor sizing if: ${{ !inputs.upload-artifact && matrix.job.arch == 'aarch64' }} working-directory: flutter - run: flutter test --timeout 60s test/cursor_size_test.dart + run: flutter test --timeout 60s test/cursor_size_test.dart test/cursor_replay_test.dart test/cursor_default_test.dart - name: Setup vcpkg with Github Actions binary cache uses: lukka/run-vcpkg@b1a0dd252f06b9e25b3c022a9a03bd7a427fb6a2 # v11 diff --git a/Cargo.lock b/Cargo.lock index c469eb81d..727ef11d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7196,6 +7196,7 @@ dependencies = [ "wol-rs", "x11-clipboard 0.8.1", "x11rb 0.12.0", + "xxhash-rust", "zstd", ] @@ -10806,6 +10807,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + [[package]] name = "yasna" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index 9fedd8cc1..76461d696 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,7 @@ lazy_static = "1.4" sha2 = "0.10" repng = "0.2" zstd = "0.13" +xxhash-rust = { version = "0.8", features = ["xxh3"] } parity-tokio-ipc = { git = "https://github.com/rustdesk-org/parity-tokio-ipc" } magnum-opus = { git = "https://github.com/rustdesk-org/magnum-opus" } dasp = { version = "0.11", features = ["signal", "interpolate-linear", "interpolate"], optional = true } diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index 30e3849c1..6653dddf8 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -58,7 +58,8 @@ const _restartReconnectSilentDelaySecs = 5; class CachedPeerData { Map updatePrivacyMode = {}; Map peerInfo = {}; - List> cursorDataList = []; + // No shapes: the core keeps them for the session, and the window a tab moves to asks it + // for the one in use. Map lastCursorId = {}; Map permissions = {}; @@ -73,7 +74,6 @@ class CachedPeerData { return jsonEncode({ 'updatePrivacyMode': updatePrivacyMode, 'peerInfo': peerInfo, - 'cursorDataList': cursorDataList, 'lastCursorId': lastCursorId, 'permissions': permissions, 'secure': secure, @@ -88,9 +88,6 @@ class CachedPeerData { final data = CachedPeerData(); data.updatePrivacyMode = map['updatePrivacyMode']; data.peerInfo = map['peerInfo']; - for (final cursorData in map['cursorDataList']) { - data.cursorDataList.add(cursorData); - } data.lastCursorId = map['lastCursorId']; map['permissions'].forEach((key, value) { data.permissions[key] = value; @@ -323,10 +320,6 @@ class FfiModel with ChangeNotifier { updatePrivacyMode(data.updatePrivacyMode, sessionId, peerId); setConnectionType(peerId, data.secure, data.direct, data.streamType); await handlePeerInfo(data.peerInfo, peerId, true); - for (final element in data.cursorDataList) { - updateLastCursorId(element); - await handleCursorData(element); - } if (data.lastCursorId.isNotEmpty) { updateLastCursorId(data.lastCursorId); handleCursorId(data.lastCursorId); @@ -356,9 +349,6 @@ class FfiModel with ChangeNotifier { } else if (name == 'switch_display') { // switch display is kept for backward compatibility handleSwitchDisplay(evt, sessionId, peerId); - } else if (name == 'cursor_data') { - updateLastCursorId(evt); - await handleCursorData(evt); } else if (name == 'cursor_id') { updateLastCursorId(evt); handleCursorId(evt); @@ -1662,9 +1652,14 @@ class FfiModel with ChangeNotifier { parent.target?.cursorModel.updateCursorId(evt); } - handleCursorData(Map evt) async { - cachedPeerData.cursorDataList.add(evt); - await parent.target?.cursorModel.updateCursorData(evt); + /// A shape arriving is the shape in use, as a cursor_id is. + handleCursorData(String id, int hotx, int hoty, int width, int height, + Uint8List colors) async { + // The replay selects this last, whatever the order the shapes are replayed in. + cachedPeerData.lastCursorId = {'id': id}; + parent.target?.cursorModel.id = id; + await parent.target?.cursorModel + .updateCursorData(id, hotx, hoty, width, height, colors); } /// Handle the peer info synchronization event based on [evt]. @@ -2858,12 +2853,13 @@ class CanvasModel with ChangeNotifier { // data for cursor class CursorData { - // At most 4 MiB of RGBA, including Linux's square cursor padding. - static const _maxRasterSize = 1024; + // At most 1 MiB of RGBA, including Linux's square cursor padding: twice the largest + // accessibility cursor, beyond which a cursor is shown smaller rather than larger. + static const _maxRasterSize = 512; final String peerId; final String id; - final img2.Image image; + img2.Image? _image; double scale; Uint8List? data; final double hotxOrigin; @@ -2879,17 +2875,28 @@ class CursorData { int get rasterWidth => _rasterWidth; int get rasterHeight => _rasterHeight; + img2.Image get image => _image!; + + /// False for a shape not in use; only the shape in use keeps pixels, for a new raster. + bool get hasPixels => _image != null; + + void releasePixels() { + _image = null; + data = null; + } + CursorData({ required this.peerId, required this.id, - required this.image, + required img2.Image image, required this.scale, required this.data, required this.hotxOrigin, required this.hotyOrigin, required this.width, required this.height, - }) : _rasterWidth = (width * scale).ceil(), + }) : _image = image, + _rasterWidth = (width * scale).ceil(), _rasterHeight = (height * scale).ceil(), hotx = hotxOrigin * scale, hoty = hotyOrigin * scale; @@ -2921,6 +2928,13 @@ class CursorData { final targetWidth = (width * scale).ceil(); final targetHeight = (height * scale).ceil(); + if ((_rasterWidth != targetWidth || _rasterHeight != targetHeight) && + !hasPixels) { + // Nothing to make this raster from; buildCursorOfCache asks for the pixels unless its + // native cursor was made before. + data = null; + return scale; + } if (_rasterWidth != targetWidth || _rasterHeight != targetHeight) { if (isWindows) { data = img2 @@ -2991,7 +3005,9 @@ class CursorData { String updateGetKey(double scale) { scale = _checkUpdateScale(scale); - return '${peerId}_${id}_${_doubleToInt(width * scale)}_${_doubleToInt(height * scale)}_${rasterWidth}_$rasterHeight'; + // The raster asked for, not the one made last: a shape without pixels keeps the native + // cursors of every raster it was shown at. + return '${peerId}_${id}_${_doubleToInt(width * scale)}_${_doubleToInt(height * scale)}_${(width * scale).ceil()}_${(height * scale).ceil()}'; } } @@ -3162,7 +3178,16 @@ class CursorModel with ChangeNotifier { get lastIsBlocked => _lastIsBlocked; - ui.Image? get image => _image; + /// The image of the shape in use, or the one shown before until it is back: a switch falls + /// back to the default cursor only for a shape the core cannot give. Asks the core for the + /// shape in use when it was let go. + ui.Image? get image { + if (_images[_id] == null && _cacheMap.containsKey(_id)) { + restorePixels(_id); + } + return _image; + } + CursorData? get cache => _cache; double get x => _x - _displayOriginX; @@ -3175,7 +3200,10 @@ class CursorModel with ChangeNotifier { double get hotx => _hotx; double get hoty => _hoty; - set id(String id) => _id = id; + set id(String id) { + if (_id != id) _unavailable = null; + _id = id; + } bool get isPeerControlProtected => DateTime.now().difference(_lastPeerMouse).inMilliseconds < @@ -3190,7 +3218,12 @@ class CursorModel with ChangeNotifier { } } - CursorModel(this.parent); + CursorModel(this.parent) { + // Made now, not when first drawn: nothing draws again when their decode lands, so the first + // build that needs one would show the shape before in its place. + preDefaultCursor.cache; + preForbiddenCursor.cache; + } Set get cachedKeys => _cacheKeys; addKey(String key) => _cacheKeys.add(key); @@ -3489,30 +3522,44 @@ class CursorModel with ChangeNotifier { _images.clear(); } - updateCursorData(Map evt) async { - final id = evt['id']; - final hotx = double.parse(evt['hotx']); - final hoty = double.parse(evt['hoty']); - final width = int.parse(evt['width']); - final height = int.parse(evt['height']); - List colors = json.decode(evt['colors']); - final rgba = Uint8List.fromList(colors.map((s) => s as int).toList()); + /// Its image is kept with the ones used last, see [_imageLimit]; a shape that does not decode + /// is marked as one the core cannot give. + Future updateCursorData(String id, int hotxInt, int hotyInt, int width, + int height, Uint8List rgba) async { + final generation = _generation; + if (_unavailable == id) _unavailable = null; + final hotx = hotxInt.toDouble(); + final hoty = hotyInt.toDouble(); final image = await img.decodeImageFromPixels( rgba, width, height, ui.PixelFormat.rgba8888); if (image == null) { + // It did not decode; painting must not ask for it on every frame. + _markUnavailable(generation, id); return; } - if (await _updateCache(rgba, image, id, hotx, hoty, width, height)) { - _images[id]?.item1.dispose(); - _images[id] = Tuple3(image, hotx, hoty); + if (!await _updateCache( + generation, rgba, image, id, hotx, hoty, width, height)) { + // Not kept, or the session was cleared while it decoded. + image.dispose(); + _markUnavailable(generation, id); + return; + } + final old = _images.remove(id); + _images[id] = Tuple3(image, hotx, hoty); + if (old != null) { + if (identical(old.item1, _image)) _image = image; + old.item1.dispose(); } // Update last cursor data. // Do not use the previous `image` and `id`, because `_id` may be changed. _updateCurData(); + final cache = _cacheMap[id]; + if (id != _id && cache != null) _switchedAway(cache); } Future _updateCache( + int generation, Uint8List rgba, ui.Image image, String id, @@ -3523,7 +3570,11 @@ class CursorModel with ChangeNotifier { ) async { Uint8List? data; img2.Image imgOrigin = img2.Image.fromBytes( - width: w, height: h, bytes: rgba.buffer, order: img2.ChannelOrder.rgba); + width: w, + height: h, + bytes: rgba.buffer, + bytesOffset: rgba.offsetInBytes, + order: img2.ChannelOrder.rgba); if (isWindows) { data = imgOrigin.getBytes(order: img2.ChannelOrder.bgra); } else { @@ -3556,34 +3607,200 @@ class CursorModel with ChangeNotifier { width: w, height: h, ); + if (generation != _generation) { + return false; + } _cacheMap[id] = cache; return true; } + @visibleForTesting + Iterable get shapeIds => _images.keys; + + @visibleForTesting + CursorData? cachedShape(String id) => _cacheMap[id]; + + final _restoring = {}; + + // Tabs in a window share its engine's native cursors, predefined ones included. + static int _nextKeyScope = 0; + final int _keyScope = _nextKeyScope++; + + String nativeKey(CursorData cache, double scale) { + final key = '${_keyScope}_${cache.updateGetKey(scale)}'; + // A native cursor at another raster does not hold the pixels this one is made from. + if (!_cacheKeys.contains(key)) _nativeIds.remove(cache.id); + return key; + } + + // Native cursors stay for the session, one per raster of each shape, and only [clear] + // deletes them, as before; unlike the painted images they have no LRU, for now. A delete + // frees nothing on Windows: the engine's `deleteCustomCursor/windows` releases the HCURSOR + // with `DeleteObject`, which does not take a cursor. Measured on Windows with 500 cursors + // made by `CreateIconIndirect`, as the engine makes them: `DeleteObject` failed all 500 and + // left 500 USER objects alive, while `DestroyCursor` or `DestroyIcon` freed all 500. Each + // delete leaks its handle until the process exits, and making the cursor again for a raster + // or a shape shown once more leaks another, so an LRU would leak more than keeping them. + // + // Fixing the engine is not cheap. x64 runs our fork, rustdesk/engine, on Flutter 3.24; the + // fix is one more patch to rebuild, publish, and carry across every Flutter upgrade. arm64 + // runs the stock engine of a newer Flutter, which has the same code: fixing it means + // porting the fork to that version and building and publishing an arm64 engine as well, or + // an upstream fix and waiting for it to reach stable. Once both engines destroy cursors, an + // LRU like the painted images' can come back. Content ids keep the count to the shapes the + // peer really shows, a few dozen to a couple of hundred. + + /// The shape in use keeps its pixels, so a new raster is made from them at once; the others + /// keep none once their native cursor holds them. + void registered(CursorData cache, String key) { + _nativeIds.add(cache.id); + _awaitingNative.remove(cache.id); + if (cache.id != _id && identical(_cacheMap[cache.id], cache)) { + cache.releasePixels(); + } + } + + // The shapes a native cursor holds, and those keeping their pixels until one does, oldest + // first: one decoded after the peer moved on must still get a native cursor when shown + // again, rather than be decoded again every time it comes back. + final _nativeIds = {}; + final _awaitingNative = {}; + + void _switchedAway(CursorData cache) { + _awaitingNative.remove(cache.id); + // Mobile only paints the cursor, from its image. + if (isMobile || _nativeIds.contains(cache.id)) { + cache.releasePixels(); + return; + } + _awaitingNative.add(cache.id); + while (_awaitingNative.length > kRecentShapes) { + final oldest = _awaitingNative.first; + _awaitingNative.remove(oldest); + if (oldest != _id) _cacheMap[oldest]?.releasePixels(); + } + } + + /// The peer's shapes used last that keep a painted image, and the most that keep their pixels + /// waiting for a native cursor; the core rebuilds the others. An animated cursor is a shape per + /// frame, 18 for the Windows busy cursor and 23 for KDE's, and a cycle longer than this limit + /// would rebuild every frame. + static const kRecentShapes = 64; + + /// The native cursor of the shape shown last, shown on while the shape in use is made. + String? shownKey; + + void shown(CursorData cache, String key) { + // The forbidden cursor shows only while input is off; it is no stand-in for a shape. + if (cache.id != kPreForbiddenCursorId) shownKey = key; + } + + /// Whether the desktop paints the peer's cursor over the remote image. + @protected + bool get showsRemoteCursor { + final tag = ShowRemoteCursorState.tag(parent.target?.id ?? ''); + return Get.isRegistered(tag: tag) && + Get.find(tag: tag).value; + } + + // Painted images kept: mobile always paints the cursor, a desktop only when it shows the + // remote cursor, and otherwise the native cursors show the shapes. + int get _imageLimit => isMobile || showsRemoteCursor ? kRecentShapes : 1; + + // `_images` is in order of use. The image painted is kept, whoever's it is. + void _evictImages() { + var excess = _images.length - _imageLimit; + _images.removeWhere((id, shape) { + if (excess <= 0 || id == _id || identical(shape.item1, _image)) { + return false; + } + excess--; + shape.item1.dispose(); + return true; + }); + } + + /// The core keeps every shape the peer sent, compressed; see `Session::cursor_shapes`. + @protected + Future fetchCursorShape(String id) { + final ffi = parent.target; + if (ffi == null) return Future.value(null); + return bind.sessionGetCursorShape(sessionId: ffi.sessionId, id: id); + } + + // The shape the core could not give, not asked for again while it is the one in use. One + // is enough: only the shape in use is asked for on every frame, and the peer's ids must not + // accumulate here. + String? _unavailable; + // Counts session clears, so that a decode finishing after one keeps nothing. + int _generation = 0; + + /// Decodes the shape in use again from the core, for a raster its native cursor lacks, for + /// painting it, or for a window a tab moved to. + void restorePixels(String id) { + if (id != _id || _unavailable == id || !_restoring.add(id)) return; + final generation = _generation; + () async { + try { + // A throw becomes an error the await hands over later, not during the build that may + // have asked: a failure tells the listeners. + final shape = await Future.sync(() => fetchCursorShape(id)); + if (generation != _generation) { + return; + } else if (shape == null) { + _markUnavailable(generation, id); + debugPrint('Cursor $id is not kept by the core'); + } else { + // Decoded even if the peer moved on: an animation comes back to it. + await updateCursorData(id, shape.hotx, shape.hoty, shape.width, + shape.height, shape.colors); + } + } catch (e) { + _markUnavailable(generation, id); + debugPrint('Failed to fetch cursor $id: $e'); + } finally { + _restoring.remove(id); + } + }(); + } + + void _markUnavailable(int generation, String id) { + if (generation != _generation || id != _id) return; + _unavailable = id; + _updateCurData(); + } + bool _updateCurData() { - _cache = _cacheMap[_id]; - final tmp = _images[_id]; + final previous = _cache; + final cache = _cacheMap[_id]; + // A shape not decoded yet leaves the one shown before in place until it is; one the core + // cannot give leaves none, so the default cursor shows, as for a shape never sent. + if (cache != null || _unavailable == _id) _cache = cache; + if (previous != null && !identical(previous, _cache)) { + _switchedAway(previous); + } + final tmp = _images.remove(_id); if (tmp != null) { + _images[_id] = tmp; _image = tmp.item1; _hotx = tmp.item2; _hoty = tmp.item3; - try { - // may throw exception, because the listener maybe already dispose - notifyListeners(); - } catch (e) { - debugPrint( - 'WARNING: updateCursorId $_id, without notifyListeners(). $e'); - } - return true; - } else { - return false; + } else if (_unavailable == _id) { + _image = null; } + _evictImages(); + try { + // may throw exception, because the listener maybe already dispose + notifyListeners(); + } catch (e) { + debugPrint('WARNING: updateCursorId $_id, without notifyListeners(). $e'); + } + return tmp != null || cache != null; } updateCursorId(Map evt) { if (!_updateCurData()) { - debugPrint( - 'WARNING: updateCursorId $_id, cache is ${_cache == null ? "null" : "not null"}. without notifyListeners()'); + restorePixels(_id); } } @@ -3636,6 +3853,10 @@ class CursorModel with ChangeNotifier { _clearCache(); _cache = null; _cacheMap.clear(); + _nativeIds.clear(); + _awaitingNative.clear(); + _unavailable = null; + _generation++; } _clearCache() { @@ -3644,6 +3865,8 @@ class CursorModel with ChangeNotifier { debugPrint("deleting cursor with key $k"); deleteCustomCursor(k); } + _cacheKeys.clear(); + shownKey = null; resetSystemCursor(); } @@ -3983,6 +4206,7 @@ class FFI { } if (isWeb) { + platformFFI.setCursorDataCallback(ffiModel.handleCursorData); platformFFI.setRgbaCallback((int display, Uint8List data) { onEvent2UIRgba(); imageModel.onRgba(display, data); @@ -4065,6 +4289,9 @@ class FFI { } else { platformFFI.nextRgba(sessionId, display); } + } else if (message is EventToUI_Cursor) { + await ffiModel.handleCursorData(message.id, message.hotx, + message.hoty, message.width, message.height, message.colors); } else if (message is EventToUI_Texture) { final display = message.field0; final gpuTexture = message.field1; diff --git a/flutter/lib/models/native_model.dart b/flutter/lib/models/native_model.dart index 93f06d55d..d56d58f38 100644 --- a/flutter/lib/models/native_model.dart +++ b/flutter/lib/models/native_model.dart @@ -285,6 +285,10 @@ class PlatformFFI { void setRgbaCallback(void Function(int, Uint8List) fun) async {} + // web only + void setCursorDataCallback( + void Function(String, int, int, int, int, Uint8List) fun) async {} + // web only, decoded WebCodecs frames arriving as ready-made images void setVideoFrameCallback( Future Function(int, ui.Image, bool Function()) fun) {} diff --git a/flutter/lib/models/web_model.dart b/flutter/lib/models/web_model.dart index be8d83500..2941fbacf 100644 --- a/flutter/lib/models/web_model.dart +++ b/flutter/lib/models/web_model.dart @@ -189,6 +189,17 @@ class PlatformFFI { }; } + // The web core calls this with a shape's pixels as bytes, as it calls onRgba for a frame. + void setCursorDataCallback( + void Function(String, int, int, int, int, Uint8List) fun) { + context["onCursorData"] = (String id, int hotx, int hoty, int width, + int height, Uint8List? rgba) { + if (rgba != null) { + fun(id, hotx, hoty, width, height, rgba); + } + }; + } + late final WebVideoFrameQueue _videoFrameQueue; // Zero-readback video path: the JS decoder hands decoded VideoFrames here diff --git a/flutter/lib/native/custom_cursor.dart b/flutter/lib/native/custom_cursor.dart index 5ea6a3951..644f83b78 100644 --- a/flutter/lib/native/custom_cursor.dart +++ b/flutter/lib/native/custom_cursor.dart @@ -16,14 +16,15 @@ resetSystemCursor() {} MouseCursor buildCursorOfCache( CursorModel cursor, double scale, CursorData? cache) { if (cache == null) { - return MouseCursor.defer; + return _shownCursor(cursor); } else { - final key = cache.updateGetKey(scale); + final key = cursor.nativeKey(cache, scale); if (!cursor.cachedKeys.contains(key)) { // data should be checked here, because it may be changed after `updateGetKey()` final data = cache.data; if (data == null) { - return MouseCursor.defer; + cursor.restorePixels(cache.id); + return _shownCursor(cursor); } // Square canvases avoid clipping or stray edge pixels on Linux. final width = isLinux && cache.rasterWidth < cache.rasterHeight @@ -48,11 +49,21 @@ MouseCursor buildCursorOfCache( ..hotX = cache.hotx ..hotY = cache.hoty); cursor.addKey(key); + cursor.registered(cache, key); } + cursor.shown(cache, key); return FlutterCustomMemoryImageCursor(key: key); } } +// The last shape's cursor stays while the one in use is made, rather than the system one. +MouseCursor _shownCursor(CursorModel cursor) { + final key = cursor.shownKey; + return key == null + ? MouseCursor.defer + : FlutterCustomMemoryImageCursor(key: key); +} + Uint8List _padCursor(Uint8List data, int size) { final bitmap = img.decodePng(data); if (bitmap == null) { diff --git a/flutter/lib/web/bridge.dart b/flutter/lib/web/bridge.dart index 6c2c23f57..f8ae3b931 100644 --- a/flutter/lib/web/bridge.dart +++ b/flutter/lib/web/bridge.dart @@ -26,6 +26,14 @@ sealed class EventToUI { int field0, bool field1, ) = EventToUI_Texture; + const factory EventToUI.cursor({ + required String id, + required int hotx, + required int hoty, + required int width, + required int height, + required Uint8List colors, + }) = EventToUI_Cursor; } class EventToUI_Event implements EventToUI { @@ -50,7 +58,60 @@ class EventToUI_Texture implements EventToUI { bool get field1 => f1; } +class EventToUI_Cursor implements EventToUI { + const EventToUI_Cursor({ + required this.id, + required this.hotx, + required this.hoty, + required this.width, + required this.height, + required this.colors, + }); + final String id; + final int hotx; + final int hoty; + final int width; + final int height; + final Uint8List colors; +} + +class CursorShape { + final int hotx; + final int hoty; + final int width; + final int height; + final Uint8List colors; + + const CursorShape({ + required this.hotx, + required this.hoty, + required this.width, + required this.height, + required this.colors, + }); +} + class RustdeskImpl { + // The core answers through the callback, before callMethod returns. + Future sessionGetCursorShape( + {required UuidValue sessionId, required String id, dynamic hint}) { + final completer = Completer(); + js.context.callMethod('getCursorShape', [ + id, + (int hotx, int hoty, int width, int height, Uint8List? colors) { + completer.complete(colors == null + ? null + : CursorShape( + hotx: hotx, + hoty: hoty, + width: width, + height: height, + colors: colors)); + } + ]); + return completer.future; + } + Future stopGlobalEventStream({required String appType, dynamic hint}) { throw UnimplementedError("stopGlobalEventStream"); } diff --git a/flutter/lib/web/custom_cursor.dart b/flutter/lib/web/custom_cursor.dart index 54df77e98..396a4ece0 100644 --- a/flutter/lib/web/custom_cursor.dart +++ b/flutter/lib/web/custom_cursor.dart @@ -102,14 +102,15 @@ resetSystemCursor() => CursorManager.instance.resetSystemCursor(); MouseCursor buildCursorOfCache( model.CursorModel cursor, double scale, model.CursorData? cache) { if (cache == null) { - return MouseCursor.defer; + return _shownCursor(cursor); } else { - final key = cache.updateGetKey(scale); + final key = cursor.nativeKey(cache, scale); if (!cursor.cachedKeys.contains(key)) { // data should be checked here, because it may be changed after `updateGetKey()` final data = cache.data; if (data == null) { - return MouseCursor.defer; + cursor.restorePixels(cache.id); + return _shownCursor(cursor); } debugPrint( "Register custom cursor with key $key (${cache.hotx},${cache.hoty})"); @@ -121,7 +122,17 @@ MouseCursor buildCursorOfCache( hotX: cache.hotx, hotY: cache.hoty)); cursor.addKey(key); + cursor.registered(cache, key); } + cursor.shown(cache, key); return FlutterCustomMemoryImageCursor(key: key); } } + +// The last shape's cursor stays while the one in use is made, rather than the system one. +MouseCursor _shownCursor(model.CursorModel cursor) { + final key = cursor.shownKey; + return key == null + ? MouseCursor.defer + : FlutterCustomMemoryImageCursor(key: key); +} diff --git a/flutter/test/cursor_default_test.dart b/flutter/test/cursor_default_test.dart new file mode 100644 index 000000000..1357a6802 --- /dev/null +++ b/flutter/test/cursor_default_test.dart @@ -0,0 +1,66 @@ +import 'dart:io'; + +import 'package:flutter/services.dart'; +import 'package:flutter_custom_cursor/flutter_custom_cursor.dart'; +import 'package:flutter_hbb/common.dart'; +import 'package:flutter_hbb/generated_bridge.dart' show CursorShape; +import 'package:flutter_hbb/models/model.dart'; +import 'package:flutter_hbb/native/custom_cursor.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:uuid/uuid.dart'; + +// A file of its own: nothing else in this isolate has made the default cursor before the model. + +/// A core that keeps no shape. +class _Cursor extends CursorModel { + _Cursor(FFI ffi) : super(WeakReference(ffi)); + @override + Future fetchCursorShape(String id) => Future.value(null); + @override + bool get showsRemoteCursor => false; +} + +class _FFI extends Fake implements FFI { + _FFI() { + ffiModel = FfiModel(WeakReference(this)); + cursorModel = _Cursor(this); + } + @override + final SessionID sessionId = const Uuid().v4obj(); + @override + late final FfiModel ffiModel; + @override + late final CursorModel cursorModel; +} + +String? _key(MouseCursor cursor) => + cursor is FlutterCustomMemoryImageCursor ? cursor.key : null; + +void main() { + final binding = TestWidgetsFlutterBinding.ensureInitialized(); + final channel = Platform.isWindows + ? SystemChannels.mouseCursor + : const MethodChannel('flutter_custom_cursor'); + binding.defaultBinaryMessenger.setMockMethodCallHandler(channel, + (call) async => (call.arguments as Map)['name']); + + test('a shape the core cannot give shows the default cursor at once', + () async { + final ffi = _FFI(); + final cursor = ffi.cursorModel; + final pixels = Uint8List.fromList(List.generate(8 * 8 * 4, (i) => i % 256)); + await ffi.ffiModel.handleCursorData('1', 0, 0, 8, 8, pixels); + final shown = _key(buildCursorOfCache(cursor, 1.0, cursor.cache)); + final evt = {'id': '2'}; + ffi.ffiModel.updateLastCursorId(evt); + ffi.ffiModel.handleCursorId(evt); + // Not a wait for the default cursor itself: asking for it would make it. + await Future.delayed(const Duration(milliseconds: 500)); + expect(cursor.cache, isNull); + final built = _key(buildCursorOfCache( + cursor, 1.0, cursor.cache ?? preDefaultCursor.cache)); + expect(built, isNot(shown)); + expect(built, contains('_${kPreDefaultCursorId}_')); + cursor.disposeImages(); + }); +} diff --git a/flutter/test/cursor_replay_test.dart b/flutter/test/cursor_replay_test.dart new file mode 100644 index 000000000..f24151a62 --- /dev/null +++ b/flutter/test/cursor_replay_test.dart @@ -0,0 +1,683 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:flutter/services.dart'; +import 'package:flutter_custom_cursor/flutter_custom_cursor.dart'; +import 'package:flutter_hbb/common.dart'; +import 'package:flutter_hbb/generated_bridge.dart' show CursorShape; +import 'package:flutter_hbb/models/model.dart'; +import 'package:flutter_hbb/native/custom_cursor.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:uuid/uuid.dart'; + +/// Stands in for the core, which keeps every shape the peer sent. +class _Cursor extends CursorModel { + _Cursor(FFI ffi) : super(WeakReference(ffi)); + final core = {}; + final fetched = []; + // While set, a fetch answers only when [answerFetches] is called. + bool holdFetches = false; + // While set, a fetch throws before it returns a future. + bool throwAtOnce = false; + final _held = , String>{}; + @override + Future fetchCursorShape(String id) { + fetched.add(id); + if (throwAtOnce) throw StateError('no core'); + if (!holdFetches) return Future.value(core[id]); + final answer = Completer(); + _held[answer] = id; + return answer.future; + } + + void answerFetches() { + final held = Map.of(_held); + _held.clear(); + held.forEach((answer, id) => answer.complete(core[id])); + } + + void failFetches() { + final held = Map.of(_held); + _held.clear(); + held.forEach((answer, _) => answer.completeError(StateError('failed'))); + } + + bool showRemoteCursor = false; + @override + bool get showsRemoteCursor => showRemoteCursor; +} + +String? _key(MouseCursor cursor) => + cursor is FlutterCustomMemoryImageCursor ? cursor.key : null; + +class _FFI extends Fake implements FFI { + _FFI() { + ffiModel = FfiModel(WeakReference(this)); + cursorModel = _Cursor(this); + } + @override + final SessionID sessionId = const Uuid().v4obj(); + @override + late final FfiModel ffiModel; + @override + late final CursorModel cursorModel; + _Cursor get cursor => cursorModel as _Cursor; +} + +Uint8List _pixels(int size, int seed) => + Uint8List.fromList(List.generate(size * size * 4, (i) => (i + seed) % 256)); + +/// A shape as the core delivers it, and keeps it. +Future _feed(_FFI ffi, String id, {int size = 8, int seed = 0}) { + final pixels = _pixels(size, seed); + ffi.cursor.core[id] = + CursorShape(hotx: 0, hoty: 0, width: size, height: size, colors: pixels); + return ffi.ffiModel.handleCursorData(id, 0, 0, size, size, pixels); +} + +void _select(_FFI ffi, String id) { + final evt = {'id': id}; + ffi.ffiModel.updateLastCursorId(evt); + ffi.ffiModel.handleCursorId(evt); +} + +Future _settle() async { + for (var i = 0; i < 2; i++) { + await Future.delayed(const Duration(milliseconds: 5)); + } +} + +void main() { + final binding = TestWidgetsFlutterBinding.ensureInitialized(); + final channel = Platform.isWindows + ? SystemChannels.mouseCursor + : const MethodChannel('flutter_custom_cursor'); + final registered = []; + final deleted = []; + late _FFI ffi; + setUp(() { + registered.clear(); + deleted.clear(); + binding.defaultBinaryMessenger.setMockMethodCallHandler(channel, + (call) async { + final args = call.arguments as Map; + if (call.method.startsWith('createCustomCursor')) { + registered.add(args['name'] as String); + return args['name']; + } + if (call.method.startsWith('deleteCustomCursor')) { + deleted.add(args['name'] as String); + } + return null; + }); + ffi = _FFI(); + }); + tearDown(() { + ffi.cursorModel.disposeImages(); + binding.defaultBinaryMessenger.setMockMethodCallHandler(channel, null); + }); + + test( + 'only the shape in use keeps an image, and a shape its native cursor holds ' + 'keeps no pixels', () async { + final cursor = ffi.cursorModel; + for (var round = 0; round < 3; round++) { + for (final (id, seed) in [('1', 1), ('2', 2), ('3', 3)]) { + await _feed(ffi, id, seed: seed); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + } + } + expect(ffi.cursorModel.shapeIds, ['3']); + for (final id in ['1', '2']) { + expect(ffi.cursorModel.cachedShape(id)!.hasPixels, isFalse); + expect(ffi.cursorModel.cachedShape(id)!.data, isNull); + } + expect(ffi.cursor.fetched, isEmpty); + }); + + test('the shape in use keeps its pixels, and a new raster is made at once', + () async { + final cursor = ffi.cursorModel; + await _feed(ffi, '1', size: 32); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + expect(cursor.cache!.hasPixels, isTrue); + expect(buildCursorOfCache(cursor, 0.5, cursor.cache), + isNot(MouseCursor.defer)); + await _settle(); + expect(ffi.cursor.fetched, isEmpty, reason: 'nothing had to be decoded'); + expect(registered.length, 2); + expect((cursor.cache!.rasterWidth, cursor.cache!.rasterHeight), (16, 16)); + + await _feed(ffi, '2'); + expect(cursor.cachedShape('1')!.hasPixels, isFalse, + reason: 'switched away from, a shape keeps no pixels'); + _select(ffi, '1'); + expect(buildCursorOfCache(cursor, 0.5, cursor.cache), + isNot(MouseCursor.defer)); + await _settle(); + expect(registered.length, 2, reason: 'its native cursor is still there'); + expect(ffi.cursor.fetched, isEmpty); + }); + + test('native cursors stay for the session, every raster of every shape', + () async { + // A delete frees nothing on Windows, so none is deleted before the session ends. + final cursor = ffi.cursorModel; + for (var i = 0; i < 100; i++) { + await _feed(ffi, '$i', size: 32); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + } + expect(deleted, isEmpty); + expect(cursor.cachedKeys.length, 100); + + _select(ffi, '0'); + expect(_key(buildCursorOfCache(cursor, 1.0, cursor.cache)), + cursor.nativeKey(cursor.cache!, 1.0)); + await _settle(); + expect(ffi.cursor.fetched, isEmpty, reason: 'its cursor is still there'); + expect(registered.length, 100); + }); + + test( + 'a raster made before is found again after the shape was switched away from', + () async { + final cursor = ffi.cursorModel; + await _feed(ffi, 'A', size: 32); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + final first = registered.single; + buildCursorOfCache(cursor, 0.5, cursor.cache); + await _settle(); + await _feed(ffi, 'B'); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + final count = registered.length; + + _select(ffi, 'A'); // its pixels went; its cursors at both rasters stay + expect(_key(buildCursorOfCache(cursor, 1.0, cursor.cache)), first); + await _settle(); + expect(ffi.cursor.fetched, isEmpty); + expect(registered.length, count); + }); + + test('a new raster of a shape without pixels keeps the cursor shown before', + () async { + final cursor = ffi.cursorModel; + await _feed(ffi, '1', size: 32); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + await _feed(ffi, '2', size: 32); + final shown = _key(buildCursorOfCache(cursor, 1.0, cursor.cache)); + await _settle(); + _select(ffi, '1'); // its native cursor held its pixels, so they went + expect(_key(buildCursorOfCache(cursor, 0.5, cursor.cache)), shown); + await _settle(); + expect(ffi.cursor.fetched, ['1']); + expect(_key(buildCursorOfCache(cursor, 0.5, cursor.cache)), + cursor.nativeKey(cursor.cache!, 0.5)); + }); + + test('shapes that keep decoding late still get their native cursors', + () async { + final cursor = ffi.cursorModel; + await _feed(ffi, 'arrow'); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + // A and B arrive, each decoding only after the peer moved on. + for (final (id, seed) in [('A', 1), ('B', 2)]) { + final pixels = _pixels(8, seed); + ffi.cursor.core[id] = + CursorShape(hotx: 0, hoty: 0, width: 8, height: 8, colors: pixels); + final decoding = ffi.ffiModel.handleCursorData(id, 0, 0, 8, 8, pixels); + _select(ffi, 'arrow'); + await decoding; + } + // Each fetch answers only after the next switch. + ffi.cursor.holdFetches = true; + for (var i = 0; i < 6; i++) { + _select(ffi, i.isEven ? 'A' : 'B'); + ffi.cursor.answerFetches(); + await _settle(); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + } + expect(registered.where((key) => key.contains('_A_')), isNotEmpty); + expect(registered.where((key) => key.contains('_B_')), isNotEmpty); + }); + + test('a zoom still gets native cursors for shapes that keep decoding late', + () async { + final cursor = ffi.cursorModel; + // An animation whose frames have native cursors at 1.0 only. + for (final (id, seed) in [('A', 1), ('B', 2)]) { + await _feed(ffi, id, size: 32, seed: seed); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + } + // At 0.5 each frame is fetched again, and each fetch answers after the next switch. + ffi.cursor.holdFetches = true; + for (var i = 0; i < 6; i++) { + _select(ffi, i.isEven ? 'A' : 'B'); + ffi.cursor.answerFetches(); + await _settle(); + buildCursorOfCache(cursor, 0.5, cursor.cache); + await _settle(); + } + expect( + registered + .where((key) => key.contains('_A_') && key.endsWith('_16_16')), + isNotEmpty); + expect( + registered + .where((key) => key.contains('_B_') && key.endsWith('_16_16')), + isNotEmpty); + }); + + test( + 'a raster still missing keeps a late restore, whatever was shown meanwhile', + () async { + final cursor = ffi.cursorModel; + await _feed(ffi, 'A', size: 32); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + await _feed(ffi, 'B'); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + + _select(ffi, 'A'); + ffi.cursor.holdFetches = true; + buildCursorOfCache(cursor, 0.5, cursor.cache); // no cursor at 0.5: fetched + buildCursorOfCache(cursor, 1.0, cursor.cache); // 1.0 has one meanwhile + _select(ffi, 'B'); + ffi.cursor.answerFetches(); + await _settle(); + ffi.cursor.fetched.clear(); + + _select(ffi, 'A'); + buildCursorOfCache(cursor, 0.5, cursor.cache); + await _settle(); + expect(ffi.cursor.fetched, isEmpty, + reason: 'its pixels waited for the cursor at 0.5'); + expect( + registered + .where((key) => key.contains('_A_') && key.endsWith('_16_16')), + isNotEmpty); + }); + + test('shapes waiting for a native cursor keep their pixels, within the limit', + () async { + const max = CursorModel.kRecentShapes; + for (var i = 0; i <= max + 1; i++) { + await _feed(ffi, '$i'); + } + expect(ffi.cursorModel.cachedShape('0')!.hasPixels, isFalse, + reason: 'the one waiting longest is let go'); + expect(ffi.cursorModel.cachedShape('1')!.hasPixels, isTrue); + expect(ffi.cursorModel.cachedShape('$max')!.hasPixels, isTrue); + }); + + test('two animated cursors and the everyday set keep their native cursors', + () async { + // Each frame is a shape of its own: 18 for the Windows busy cursor, 23 for + // KDE Breeze's wait and progress cursors on X11. + const frames = 23; + const statics = 12; + final cursor = ffi.cursorModel; + final shapes = [ + for (var i = 0; i < statics; i++) 's$i', + for (var i = 0; i < frames; i++) 'wait$i', + for (var i = 0; i < frames; i++) 'progress$i', + ]; + for (var i = 0; i < shapes.length; i++) { + await _feed(ffi, shapes[i], seed: i); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + } + ffi.cursor.fetched.clear(); + for (final id in shapes) { + _select(ffi, id); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + } + expect(ffi.cursor.fetched, isEmpty, + reason: 'every shape kept its native cursor'); + expect(registered.length, shapes.length); + expect(deleted, isEmpty); + }); + + test('a shape painted again is decoded again, the last one shown meanwhile', + () async { + await _feed(ffi, '1', size: 16); + await _feed(ffi, '2'); + _select(ffi, '1'); + expect(ffi.cursorModel.image?.width, 8, reason: 'shape 2 until 1 is back'); + await _settle(); + expect(ffi.cursor.fetched, ['1']); + expect(ffi.cursorModel.image?.width, 16); + expect(ffi.cursorModel.shapeIds, ['1']); + }); + + test('a shape the core gives but that does not decode is asked for once', + () async { + await _feed(ffi, '1'); + await _feed(ffi, '2'); + ffi.cursor.core['1'] = CursorShape( + hotx: 0, hoty: 0, width: 8, height: 8, colors: Uint8List(3)); + _select(ffi, '1'); + for (var i = 0; i < 5; i++) { + ffi.cursorModel.image; + await _settle(); + } + expect(ffi.cursor.fetched, ['1']); + }); + + test('a shape switched away from while it decoded is asked for again', + () async { + await _feed(ffi, '1', size: 16); + await _feed(ffi, '2'); + _select(ffi, '1'); + ffi.cursorModel.image; // asks the core + // The core has answered and the shape is decoding when the peer moves on. + await Future.delayed(Duration.zero); + _select(ffi, '2'); + await _settle(); + expect(ffi.cursorModel.cachedShape('1')!.hasPixels, isTrue, + reason: + 'decoded after the peer moved on, it waits for its native cursor'); + _select(ffi, '1'); + ffi.cursorModel.image; + await _settle(); + expect(ffi.cursor.fetched, ['1', '1']); + expect(ffi.cursorModel.image?.width, 16); + }); + + test('a shape decoded after the session was cleared is not kept', () async { + await _feed(ffi, '1', size: 16); + await _feed(ffi, '2'); + _select(ffi, '1'); + ffi.cursorModel.image; // asks the core + ffi.cursorModel.clear(); + await _settle(); + expect(ffi.cursorModel.shapeIds, isEmpty); + expect(ffi.cursorModel.cache, isNull); + }); + + test('a shape the core does not have is asked for once', () async { + await _feed(ffi, '1'); + await _feed(ffi, '2'); + ffi.cursor.core.remove('1'); + _select(ffi, '1'); + for (var i = 0; i < 5; i++) { + ffi.cursorModel.image; + await _settle(); + } + expect(ffi.cursor.fetched, ['1']); + }); + + test('a shape the core lacks is remembered only while it is in use', + () async { + await _feed(ffi, '1'); + for (var i = 0; i < 1000; i++) { + _select(ffi, 'missing$i'); + ffi.cursorModel.image; + } + await _settle(); + expect(ffi.cursor.fetched.length, 1000, reason: 'each is asked for once'); + _select(ffi, 'missing0'); + await _settle(); + expect(ffi.cursor.fetched.length, 1001, + reason: 'nothing is kept for a shape no longer in use'); + }); + + test('a shape the core lacks is asked for again after another was in use', + () async { + await _feed(ffi, '1'); + _select(ffi, 'missing0'); + await _settle(); + _select(ffi, '1'); + _select(ffi, 'missing0'); + await _settle(); + expect(ffi.cursor.fetched, ['missing0', 'missing0']); + }); + + test('a moved tab carries the id in use, and its window asks the core', + () async { + await _feed(ffi, '1', size: 16); + await _feed(ffi, '2'); + await _feed(ffi, '1', size: 16); + final carried = ffi.ffiModel.cachedPeerData.toString(); + expect(carried, isNot(contains('colors'))); + expect(CachedPeerData.fromString(carried)!.lastCursorId['id'], '1', + reason: 'a shape arriving is the shape in use'); + + final moved = _FFI(); + addTearDown(moved.cursorModel.disposeImages); + moved.cursor.core.addAll(ffi.cursor.core); + _select(moved, '1'); + await _settle(); + expect(moved.cursor.fetched, ['1']); + expect(moved.cursorModel.cache?.id, '1'); + expect(moved.cursorModel.image?.width, 16); + }); + + test('a shape keeps a native cursor for each raster it was shown at', + () async { + final cursor = ffi.cursorModel; + await _feed(ffi, '1', size: 32); + for (final scale in [1.0, 0.5, 1.0, 0.5]) { + buildCursorOfCache(cursor, scale, cursor.cache); + await _settle(); + } + expect(registered.length, 2, reason: 'a raster returned to is reused'); + expect(deleted, isEmpty); + expect(cursor.cachedKeys, registered.toSet()); + }); + + test('a cleared session forgets the native cursors it deleted', () async { + final cursor = ffi.cursorModel; + await _feed(ffi, '1'); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + cursor.clear(); + await _settle(); + expect(deleted, registered); + expect(cursor.cachedKeys, isEmpty); + }); + + test('a tab closing does not delete the cursors of another tab', () async { + final other = _FFI(); + addTearDown(other.cursorModel.disposeImages); + while (preDefaultCursor.cache == null) { + await _settle(); + } + buildCursorOfCache(ffi.cursorModel, 1.0, preDefaultCursor.cache); + buildCursorOfCache(other.cursorModel, 1.0, preDefaultCursor.cache); + await _settle(); + expect(registered.toSet().length, 2, reason: 'each tab has its own'); + + ffi.cursorModel.clear(); + await _settle(); + expect(other.cursorModel.cachedKeys.single, isNot(isIn(deleted))); + }); + + test('a shape given as a view into a larger buffer is read from the view', + () async { + final pixels = _pixels(8, 3); + final padded = Uint8List(16 + pixels.length) + ..setRange(16, 16 + pixels.length, pixels); + await ffi.ffiModel + .handleCursorData('1', 0, 0, 8, 8, Uint8List.sublistView(padded, 16)); + final p = ffi.cursorModel.cache!.image.getPixel(0, 0); + expect([p.r, p.g, p.b, p.a], [3, 4, 5, 6]); + }); + + test('a shape still decoding leaves the one shown before in place', () async { + final cursor = ffi.cursorModel; + await _feed(ffi, '1'); + final shown = _key(buildCursorOfCache(cursor, 1.0, cursor.cache)); + await _settle(); + _select(ffi, '2'); // its shape has not decoded yet + expect(cursor.cache?.id, '1'); + expect(_key(buildCursorOfCache(cursor, 1.0, cursor.cache)), shown); + expect(cursor.image, isNotNull); + await _feed(ffi, '2'); + expect(cursor.cache?.id, '2'); + }); + + for (final how in ['does not have', 'cannot decode']) { + test('a shape the core $how leaves the default cursor, not the one before', + () async { + final cursor = ffi.cursorModel; + await _feed(ffi, '1'); + if (how == 'cannot decode') { + ffi.cursor.core['2'] = CursorShape( + hotx: 0, hoty: 0, width: 8, height: 8, colors: Uint8List(3)); + } + _select(ffi, '2'); + expect(cursor.cache?.id, '1', reason: 'kept while the shape may come'); + var notified = 0; + cursor.addListener(() => notified++); + await _settle(); + expect(cursor.cache, isNull, + reason: 'the desktop shows preDefaultCursor'); + expect(cursor.image, isNull, reason: 'and so does a painted cursor'); + expect(notified, greaterThan(0)); + }); + } + + test('a shape sent that does not decode leaves the default cursor', () async { + final cursor = ffi.cursorModel; + await _feed(ffi, '1'); + ffi.cursor.core['2'] = CursorShape( + hotx: 0, hoty: 0, width: 8, height: 8, colors: Uint8List(3)); + await ffi.ffiModel.handleCursorData('2', 0, 0, 8, 8, Uint8List(3)); + expect(cursor.cache, isNull); + expect(cursor.image, isNull); + _select(ffi, '2'); + await _settle(); + expect(ffi.cursor.fetched, isEmpty, reason: 'it is not asked for either'); + }); + + test('a fetch failing after the session was cleared marks nothing in it', + () async { + final cursor = ffi.cursorModel; + ffi.cursor.holdFetches = true; + _select(ffi, '1'); // not decoded here yet, so it is asked for + cursor.clear(); + ffi.cursor.failFetches(); + await _settle(); + ffi.cursor.holdFetches = false; + _select(ffi, '1'); // the same id, now in the new session + await _settle(); + expect(ffi.cursor.fetched, ['1', '1']); + }); + + test('a fetch that throws at once tells the listeners after the build', + () async { + final cursor = ffi.cursorModel; + await _feed(ffi, '1'); + ffi.cursor.throwAtOnce = true; + cursor.id = '2'; + var notified = 0; + cursor.addListener(() => notified++); + cursor.restorePixels('2'); + expect(notified, 0, reason: 'a build may be what asked'); + await _settle(); + expect(notified, greaterThan(0)); + expect(cursor.cache, isNull); + }); + + test('a raster still being made falls back to a shape, not the forbidden one', + () async { + final cursor = ffi.cursorModel; + for (var i = 0; preForbiddenCursor.cache == null && i < 100; i++) { + await _settle(); + } + expect(preForbiddenCursor.cache, isNotNull); + await _feed(ffi, '1', size: 32); + buildCursorOfCache(cursor, 1.0, cursor.cache); + await _settle(); + await _feed(ffi, '2', size: 32); + final shown = _key(buildCursorOfCache(cursor, 1.0, cursor.cache)); + await _settle(); + buildCursorOfCache(cursor, 1.0, preForbiddenCursor.cache); // input disabled + _select(ffi, '1'); // its native cursor held its pixels, so they went + expect(_key(buildCursorOfCache(cursor, 0.5, cursor.cache)), shown); + }); + + group('a painted cursor', () { + void painted(String how) { + if (how == 'mobile') { + final was = isMobile; + isMobile = true; + addTearDown(() => isMobile = was); + } else { + ffi.cursor.showRemoteCursor = true; + } + } + + for (final how in ['mobile', 'remote cursor shown']) { + test('keeps the images of two animations and the everyday set ($how)', + () async { + painted(how); + final cursor = ffi.cursorModel; + final shapes = [ + for (var i = 0; i < 12; i++) 's$i', + for (var i = 0; i < 23; i++) 'wait$i', + for (var i = 0; i < 23; i++) 'progress$i', + ]; + for (var i = 0; i < shapes.length; i++) { + await _feed(ffi, shapes[i], seed: i); + } + for (final id in shapes) { + _select(ffi, id); + expect(cursor.shapeIds.last, id, reason: 'its image is at hand'); + } + await _settle(); + expect(ffi.cursor.fetched, isEmpty); + }); + } + + test('keeps the images used last, and decodes the others again', () async { + painted('mobile'); + const max = CursorModel.kRecentShapes; + final cursor = ffi.cursorModel; + for (var i = 0; i <= max; i++) { + await _feed(ffi, '$i'); + } + expect(cursor.shapeIds.length, max); + _select(ffi, '$max'); + _select(ffi, '1'); + await _settle(); + expect(ffi.cursor.fetched, isEmpty, reason: 'shape 1 was used lately'); + _select(ffi, '0'); + expect(cursor.image, isNotNull, reason: 'shape 1 until 0 is back'); + await _settle(); + expect(ffi.cursor.fetched, ['0']); + expect(cursor.shapeIds.length, max); + }); + + test('keeps a shape that finished decoding after the peer moved on', + () async { + painted('mobile'); + const max = CursorModel.kRecentShapes; + final cursor = ffi.cursorModel; + for (var i = 0; i <= max; i++) { + await _feed(ffi, '$i'); + } + _select(ffi, '0'); + cursor.image; // let go, so it is asked for + await Future.delayed(Duration.zero); + _select(ffi, '1'); + await _settle(); + _select(ffi, '0'); + cursor.image; + await _settle(); + expect(ffi.cursor.fetched, ['0'], reason: 'the late one was kept'); + expect(cursor.shapeIds.last, '0'); + }); + }); +} diff --git a/flutter/test/cursor_size_test.dart b/flutter/test/cursor_size_test.dart index 7df1cf74e..254fe4766 100644 --- a/flutter/test/cursor_size_test.dart +++ b/flutter/test/cursor_size_test.dart @@ -1,4 +1,3 @@ -import 'dart:convert'; import 'dart:io'; import 'dart:ui' as ui; @@ -79,6 +78,8 @@ class _FFI extends Fake implements FFI { final ffiModel = _Peer(); @override final inputModel = _Input(); + @override + var id = ''; } class _Cursor extends CursorModel { @@ -111,7 +112,7 @@ void main() { ((1, 64), 0.25, (1, 16)), ((8, 8), 0.5, (12, 12)), ((4, 64), 1.0, (4, 64)), - ((1, 512), 10.0, (2, 1024)), + ((1, 512), 10.0, (1, 512)), ]) { test('native cursor size $scenario', () => _checkSize(scenario, registrations)); @@ -181,15 +182,14 @@ Future _checkWindowsPeerAlpha( addTearDown(() => _dispose(cursor)); addTearDown(cursor.disposeImages); addTearDown(ffi.canvasModel.dispose); - await cursor.updateCursorData({ - 'id': 'alpha', - 'hotx': '0', - 'hoty': '0', - 'width': '$sourceSize', - 'height': '$sourceSize', - 'colors': jsonEncode(List.generate(sourceSize * sourceSize * channels, - (i) => pattern.$1[i % pattern.$1.length])), - }); + await cursor.updateCursorData( + 'alpha', + 0, + 0, + sourceSize, + sourceSize, + Uint8List.fromList(List.generate(sourceSize * sourceSize * channels, + (i) => pattern.$1[i % pattern.$1.length]))); buildCursorOfCache(cursor, 1.0 / dpr, cursor.cache); await Future.delayed(Duration.zero); final args = registrations.single; @@ -259,7 +259,7 @@ Future _checkRasterTransitions( ]) { buildCursorOfCache(cursor, scale, cursor.cache); await Future.delayed(Duration.zero); - final key = cursor.cache.updateGetKey(scale); + final key = cursor.nativeKey(cursor.cache, scale); _expectSize( registrations.singleWhere((args) => args['name'] == key), expected); } @@ -268,7 +268,7 @@ Future _checkRasterTransitions( Future _checkResizeLimits( (int, int) size, List> registrations) async { - const maxSide = 1024; + const maxSide = 512; const sourceLongEdge = 30; const validScale = maxSide / sourceLongEdge; final ffi = _FFI(_Canvas(kRemoteViewStyleAdaptive)); @@ -361,7 +361,7 @@ Future _checkView(WidgetTester tester, (String, bool) mode, mode.$1 != kRemoteViewStyleOriginal ? (sourceSize * (Platform.isWindows ? dpr : 1.0)).ceil() : (size * scale * (Platform.isWindows ? dpr : 1.0)).ceil(); - final key = cursor.cache.updateGetKey(cursor.cache.scale); + final key = cursor.nativeKey(cursor.cache, cursor.cache.scale); _expectSize(registrations.singleWhere((v) => v['name'] == key), (w, w)); } await tester.pumpWidget(const SizedBox.shrink()); diff --git a/libs/base/protos/message.proto b/libs/base/protos/message.proto index dd62942be..991e8a31e 100644 --- a/libs/base/protos/message.proto +++ b/libs/base/protos/message.proto @@ -310,7 +310,9 @@ message KeyEvent { } message CursorData { - uint64 id = 1; + // A string in JS: a Windows peer before 1.5.0 sends its cursor handle, sign-extended above + // 2^53 when its top bit is set, which a JS number cannot hold. + uint64 id = 1 [jstype = JS_STRING]; sint32 hotx = 2; sint32 hoty = 3; int32 width = 4; @@ -859,6 +861,12 @@ message Misc { DisplayResolution change_display_resolution = 36; MessageQuery message_query = 37; int32 follow_current_display = 38; + // Since 1.5.0: send the CursorData of this id again. No controller sends this yet; it is + // here so a controller can one day bound its own shape store and ask for the shape the + // peer just selected. The peer answers with what it holds then: a shape its cache dropped + // is rebuilt, and held again, the next time it is shown, before any connection names it + // by id again, so the shape just selected is always there to give. + uint64 request_cursor_data = 39; } } @@ -1002,7 +1010,7 @@ message Message { AudioFrame audio_frame = 11; CursorData cursor_data = 12; CursorPosition cursor_position = 13; - uint64 cursor_id = 14; + uint64 cursor_id = 14 [jstype = JS_STRING]; // as CursorData.id KeyEvent key_event = 15; Clipboard clipboard = 16; FileAction file_action = 17; diff --git a/src/client/io_loop.rs b/src/client/io_loop.rs index 4ee6fa1bb..fcbc0901e 100644 --- a/src/client/io_loop.rs +++ b/src/client/io_loop.rs @@ -61,7 +61,7 @@ use hbb_common::{ use hbb_common::{tokio::sync::Mutex as TokioMutex, ResultType}; use scrap::CodecFormat; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, ffi::c_void, num::NonZeroI64, path::PathBuf, @@ -101,6 +101,7 @@ pub struct Remote { chroma: Arc>>, last_record_state: bool, sent_close_reason: bool, + cursor_dedupe: CursorDedupe, } #[derive(Default)] @@ -152,6 +153,7 @@ impl Remote { chroma: Default::default(), last_record_state: false, sent_close_reason: false, + cursor_dedupe: Default::default(), } } @@ -1647,11 +1649,29 @@ impl Remote { } _ => {} }, + Some(message::Union::CursorData(cd)) if self.dedupes_cursors() => { + self.set_cursor_data_by_content(cd); + } + Some(message::Union::CursorId(id)) if self.dedupes_cursors() => { + self.handler + .set_cursor_id(self.cursor_dedupe.id(id).to_string()); + } Some(message::Union::CursorData(cd)) => { let id = cd.id; + #[cfg(feature = "flutter")] + let compressed = cd.colors.clone(); match decode_cursor_data(cd) { - Ok(cd) => self.handler.set_cursor_data(cd), - Err(err) => log::warn!("Rejected cursor {id}: {err}"), + Ok(cd) => { + #[cfg(feature = "flutter")] + self.keep_cursor_shape(cursor_shape(id, &cd, &compressed)); + self.handler.set_cursor_data(cd) + } + Err(err) => { + log::warn!("Rejected cursor {id}: {err}"); + // The peer shows it now: selected, the UI treats it as a shape it + // lacks, as it will when the peer selects it again. + self.handler.set_cursor_id(id.to_string()); + } } } Some(message::Union::CursorId(id)) => { @@ -2701,6 +2721,69 @@ impl Remote { msg.set_misc(misc); self.sender.send(Data::Message(msg)).ok(); } + + /// A shape that decoded, for the UI to ask for again; see `Session::cursor_shapes`. + #[cfg(feature = "flutter")] + fn keep_cursor_shape(&self, shape: CursorData) { + self.handler + .cursor_shapes + .write() + .unwrap() + .insert(shape.id, shape); + } + + fn dedupes_cursors(&self) -> bool { + !crate::is_peer_naming_cursors_by_content(self.handler.lc.read().unwrap().version) + } + + /// A shape the UI already has, under whatever handle, is only selected. + fn set_cursor_data_by_content(&mut self, cd: CursorData) { + let peer_id = cd.id; + let id = CursorDedupe::name(&cd); + if self.cursor_dedupe.has_decoded(id) { + self.cursor_dedupe.record(peer_id, id); + self.handler.set_cursor_id(id.to_string()); + return; + } + #[cfg(feature = "flutter")] + let compressed = cd.colors.clone(); + match decode_cursor_data(cd) { + Ok(mut cd) => { + cd.id = id; + self.cursor_dedupe.record(peer_id, id); + #[cfg(feature = "flutter")] + self.keep_cursor_shape(cursor_shape(id, &cd, &compressed)); + self.handler.set_cursor_data(cd); + } + Err(err) => { + log::warn!("Rejected cursor {peer_id}: {err}"); + self.handler + .set_cursor_id(self.cursor_dedupe.rejected(peer_id, id).to_string()); + } + } + } +} + +/// A shape that decoded, still compressed, under the id the UI knows it by. Copied once it +/// decoded: the message's colors may be a slice of a larger received buffer, which a clone +/// would keep alive. +#[cfg(any(feature = "flutter", test))] +fn cursor_shape(id: u64, cd: &CursorData, compressed: &[u8]) -> CursorData { + CursorData { + id, + hotx: cd.hotx, + hoty: cd.hoty, + width: cd.width, + height: cd.height, + colors: compressed.to_vec().into(), + ..Default::default() + } +} + +/// The RGBA of a shape kept by `Session::cursor_shapes`, for the UI to draw it again. +#[cfg(any(feature = "flutter", test))] +pub(crate) fn kept_cursor_rgba(shape: CursorData) -> hbb_common::ResultType { + decode_cursor_data(shape) } // Both UI handlers receive validated, uncompressed RGBA from the receive loop. @@ -2744,6 +2827,49 @@ fn decode_cursor_data(data: CursorData) -> hbb_common::ResultType { Ok(cd) } +/// Gives the UI one id per shape for a peer that names shapes by handle, so the UI decodes and +/// keeps a shape once however many handles it arrives under. Nothing is dropped for the +/// connection: the peer sends a shape once and may select any handle it named again. +#[derive(Default)] +struct CursorDedupe { + ids: HashMap, + // The shapes given to the UI, by content id: whether one that arrives needs decoding. + decoded: HashSet, +} + +impl CursorDedupe { + /// Hashes the compressed colors: one peer compresses the same pixels to the same bytes, and + /// a shape seen before is then never decompressed again. + fn name(cd: &CursorData) -> u64 { + crate::cursor_content_id(cd.width, cd.height, cd.hotx, cd.hoty, &cd.colors) + } + + fn id(&self, peer_id: u64) -> u64 { + self.ids.get(&peer_id).copied().unwrap_or(peer_id) + } + + fn has_decoded(&self, id: u64) -> bool { + self.decoded.contains(&id) + } + + /// Names the handle once its shape decoded, now or before. + fn record(&mut self, peer_id: u64, id: u64) { + self.ids.insert(peer_id, id); + self.decoded.insert(id); + } + + /// Names a known handle by the content of a shape that did not decode, which is never + /// marked decoded: the UI has no shape under it, now or when the handle is selected again, + /// and the shape the handle named before is not shown in its place. A handle never seen is + /// not filed, so what a peer sends that does not decode takes no room. + fn rejected(&mut self, peer_id: u64, id: u64) -> u64 { + if let Some(named) = self.ids.get_mut(&peer_id) { + *named = id; + } + id + } +} + struct RemoveJob { files: Vec, path: String, @@ -2866,3 +2992,145 @@ mod tests { ); } } + +#[cfg(test)] +mod cursor_dedupe_tests { + use super::*; + + fn shape(id: u64, hotx: i32, colors: &[u8]) -> CursorData { + CursorData { + id, + hotx, + width: 2, + height: 2, + colors: colors.to_vec().into(), + ..Default::default() + } + } + + #[test] + fn a_shape_sent_under_a_new_handle_keeps_its_id() { + let mut dedupe = CursorDedupe::default(); + let first = CursorDedupe::name(&shape(1, 0, b"arrow")); + assert!(!dedupe.has_decoded(first)); + dedupe.record(1, first); + + let again = CursorDedupe::name(&shape(2, 0, b"arrow")); + assert_eq!(again, first); + assert!(dedupe.has_decoded(again)); + dedupe.record(2, again); + assert_eq!(dedupe.id(2), first); + assert_eq!(dedupe.id(1), first); + } + + #[test] + fn the_hotspot_is_part_of_the_shape() { + let a = CursorDedupe::name(&shape(1, 0, b"arrow")); + let b = CursorDedupe::name(&shape(2, 1, b"arrow")); + assert_ne!(a, b); + } + + #[test] + fn a_reused_handle_names_its_latest_shape() { + let mut dedupe = CursorDedupe::default(); + dedupe.record(1, CursorDedupe::name(&shape(1, 0, b"arrow"))); + let beam = CursorDedupe::name(&shape(1, 0, b"beam")); + dedupe.record(1, beam); + assert_eq!(dedupe.id(1), beam); + } + + #[test] + fn a_rejected_shape_names_its_handle_and_is_never_decoded() { + let mut dedupe = CursorDedupe::default(); + let arrow = CursorDedupe::name(&shape(7, 0, b"arrow")); + dedupe.record(7, arrow); + let beam = CursorDedupe::name(&shape(8, 0, b"beam")); + dedupe.record(8, beam); + let broken = CursorDedupe::name(&shape(7, 0, b"broken")); + assert_eq!( + dedupe.rejected(7, broken), + broken, + "not the shape it named before" + ); + assert_eq!( + dedupe.id(7), + broken, + "nor when the handle is selected again" + ); + assert_eq!(dedupe.id(8), beam, "another handle keeps its shape"); + assert!(!dedupe.has_decoded(broken)); + assert!( + dedupe.has_decoded(arrow), + "kept for a handle that brings it again" + ); + dedupe.record(7, arrow); + assert_eq!(dedupe.id(7), arrow); + } + + #[test] + fn a_shape_rejected_under_handles_never_seen_files_nothing() { + let mut dedupe = CursorDedupe::default(); + let broken = CursorDedupe::name(&shape(0, 0, b"broken")); + for handle in 0..1000 { + assert_eq!(dedupe.rejected(handle, broken), broken); + } + assert!(dedupe.ids.is_empty()); + assert!(!dedupe.has_decoded(broken)); + } + + #[test] + fn an_id_the_peer_never_sent_is_passed_on() { + assert_eq!(CursorDedupe::default().id(7), 7); + } + + #[test] + fn peers_from_1_5_0_name_cursors_by_content() { + use hbb_common::get_version_number as v; + assert!(!crate::is_peer_naming_cursors_by_content(v("1.4.9"))); + assert!(crate::is_peer_naming_cursors_by_content(v("1.5.0"))); + assert!(crate::is_peer_naming_cursors_by_content(v("1.5.1"))); + } +} + +#[cfg(test)] +mod kept_cursor_tests { + use super::*; + + fn compressed(width: i32, height: i32) -> CursorData { + let rgba = vec![7u8; (width * height * 4) as usize]; + CursorData { + id: 1, + width, + height, + colors: hbb_common::compress::compress(&rgba).into(), + ..Default::default() + } + } + + #[test] + fn a_kept_shape_is_a_copy_under_the_ui_id() { + let mut cd = compressed(4, 4); + cd.hotx = 1; + let kept = cursor_shape(42, &cd, &cd.colors); + assert_eq!(kept.id, 42); + assert_eq!((kept.hotx, kept.width, kept.height), (1, 4, 4)); + assert_eq!(kept.colors, cd.colors); + assert_ne!( + kept.colors.as_ptr(), + cd.colors.as_ptr(), + "not a view of the received buffer" + ); + } + + #[test] + fn a_kept_shape_is_checked_again_when_it_is_read() { + assert_eq!(kept_cursor_rgba(compressed(4, 4)).unwrap().colors.len(), 64); + assert!( + kept_cursor_rgba(compressed(513, 1)).is_err(), + "over the size cap" + ); + let mut bomb = compressed(4, 4); + bomb.colors = hbb_common::compress::compress(&vec![0u8; 1 << 20]).into(); + assert!(kept_cursor_rgba(bomb).is_err(), "more pixels than its size"); + } +} diff --git a/src/common.rs b/src/common.rs index 7daee26b8..0d46bafe2 100644 --- a/src/common.rs +++ b/src/common.rs @@ -149,6 +149,24 @@ pub fn is_support_multi_ui_session_num(ver: i64) -> bool { ver >= hbb_common::get_version_number(MIN_VER_MULTI_UI_SESSION) } +/// Peers from the 1.5.0 release name a cursor by `cursor_content_id`; older ones by the +/// platform handle, which apps mint anew for the same shape. +#[inline] +pub fn is_peer_naming_cursors_by_content(ver: i64) -> bool { + ver >= hbb_common::get_version_number("1.5.0") +} + +/// One id per cursor look, however many handles a platform gives it. Held to 53 bits: the web +/// client decodes a u64 into a JS number and drops the whole message when it does not fit. +pub fn cursor_content_id(width: i32, height: i32, hotx: i32, hoty: i32, colors: &[u8]) -> u64 { + let mut hasher = xxhash_rust::xxh3::Xxh3::new(); + for v in [width, height, hotx, hoty] { + hasher.update(&v.to_le_bytes()); + } + hasher.update(colors); + hasher.digest() & ((1 << 53) - 1) +} + #[inline] #[cfg(feature = "unix-file-copy-paste")] pub fn is_support_file_copy_paste(ver: &str) -> bool { @@ -2936,6 +2954,16 @@ mod tests { }; use std::collections::HashSet; + #[test] + fn a_cursor_content_id_fits_a_web_client_number() { + const MAX_SAFE_INTEGER: u64 = (1 << 53) - 1; + for colors in [&b"arrow"[..], b"beam", b"hand", b""] { + for hotx in 0..64 { + assert!(cursor_content_id(32, 32, hotx, 0, colors) <= MAX_SAFE_INTEGER); + } + } + } + #[inline] fn get_timestamp_secs() -> u128 { (std::time::SystemTime::UNIX_EPOCH diff --git a/src/flutter.rs b/src/flutter.rs index 345ae3929..35fa8cc5e 100644 --- a/src/flutter.rs +++ b/src/flutter.rs @@ -651,23 +651,21 @@ impl FlutterHandler { } impl InvokeUiSession for FlutterHandler { + // On the event stream, so that it keeps its order with the cursor_id events around it. fn set_cursor_data(&self, cd: CursorData) { - let colors = &cd.colors; - self.push_event( - "cursor_data", - &[ - ("id", &cd.id.to_string()), - ("hotx", &cd.hotx.to_string()), - ("hoty", &cd.hoty.to_string()), - ("width", &cd.width.to_string()), - ("height", &cd.height.to_string()), - ( - "colors", - &serde_json::ser::to_string(&colors).unwrap_or("".to_owned()), - ), - ], - &[], - ); + let colors = cd.colors.to_vec(); + for session in self.session_handlers.read().unwrap().values() { + if let Some(stream) = &session.event_stream { + stream.add(EventToUI::Cursor { + id: cd.id.to_string(), + hotx: cd.hotx, + hoty: cd.hoty, + width: cd.width, + height: cd.height, + colors: colors.clone(), + }); + } + } } fn set_cursor_id(&self, id: String) { diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index 2e869d141..f9cc2d66c 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -97,6 +97,16 @@ pub enum EventToUI { Event(String), Rgba(usize), Texture(usize, bool), // (display, gpu_texture) + // A shape's RGBA as bytes: as text it was four times the size and parsed on the UI thread. + // The id stays text, since the peer's ids can exceed Dart's int. + Cursor { + id: String, + hotx: i32, + hoty: i32, + width: i32, + height: i32, + colors: Vec, + }, } pub fn host_stop_system_key_propagate(_stopped: bool) { @@ -201,6 +211,34 @@ pub fn session_start_with_displays( Ok(()) } +// A cursor shape the core keeps, as RGBA, for a window that has to draw it again. +pub struct CursorShape { + pub hotx: i32, + pub hoty: i32, + pub width: i32, + pub height: i32, + pub colors: Vec, +} + +pub fn session_get_cursor_shape(session_id: SessionID, id: String) -> Option { + let id = id.parse::().ok()?; + let session = sessions::get_session_by_session_id(&session_id)?; + let shape = session.cursor_shapes.read().unwrap().get(&id).cloned()?; + match crate::client::io_loop::kept_cursor_rgba(shape) { + Ok(cd) => Some(CursorShape { + hotx: cd.hotx, + hoty: cd.hoty, + width: cd.width, + height: cd.height, + colors: cd.colors.to_vec(), + }), + Err(err) => { + log::warn!("Kept cursor {id} failed to decode: {err}"); + None + } + } +} + pub fn session_get_remember(session_id: SessionID) -> Option { if let Some(session) = sessions::get_session_by_session_id(&session_id) { Some(session.get_remember()) diff --git a/src/platform/macos.rs b/src/platform/macos.rs index 82bcf9d6f..4386e07ee 100644 --- a/src/platform/macos.rs +++ b/src/platform/macos.rs @@ -559,6 +559,9 @@ pub fn get_cursor() -> ResultType> { autoreleasepool(|| unsafe_get_cursor()) } +// The seed names a change of cursor, not a cursor: the shape is named by its content once +// captured, as on every platform. A fingerprint of its size, hotspot and two pixels named the open +// and the closed hand alike, and the arrows with a badge, so a switch between them went unsent. fn unsafe_get_cursor() -> ResultType> { unsafe { let seed = CGSCurrentCursorSeed(); @@ -566,9 +569,8 @@ fn unsafe_get_cursor() -> ResultType> { return Ok(None); } LATEST_SEED = seed; + Ok(Some(seed as u32 as u64)) } - let c = get_cursor_id()?; - Ok(Some(c.1)) } pub fn reset_input_cache() { @@ -577,53 +579,6 @@ pub fn reset_input_cache() { } } -fn get_cursor_id() -> ResultType<(id, u64)> { - unsafe { - let c: id = msg_send![class!(NSCursor), currentSystemCursor]; - if c == nil { - bail!("Failed to call [NSCursor currentSystemCursor]"); - } - let hotspot: NSPoint = msg_send![c, hotSpot]; - let img: id = msg_send![c, image]; - if img == nil { - bail!("Failed to call [NSCursor image]"); - } - let size: NSSize = msg_send![img, size]; - let tif: id = msg_send![img, TIFFRepresentation]; - if tif == nil { - bail!("Failed to call [NSImage TIFFRepresentation]"); - } - let rep: id = msg_send![class!(NSBitmapImageRep), imageRepWithData: tif]; - if rep == nil { - bail!("Failed to call [NSBitmapImageRep imageRepWithData]"); - } - let rep_size: NSSize = msg_send![rep, size]; - let mut hcursor = - size.width + size.height + hotspot.x + hotspot.y + rep_size.width + rep_size.height; - let x = (rep_size.width * hotspot.x / size.width) as usize; - let y = (rep_size.height * hotspot.y / size.height) as usize; - for i in 0..2 { - let mut x2 = x + i; - if x2 >= rep_size.width as usize { - x2 = rep_size.width as usize - 1; - } - let mut y2 = y + i; - if y2 >= rep_size.height as usize { - y2 = rep_size.height as usize - 1; - } - let color: id = msg_send![rep, colorAtX:x2 y:y2]; - if color != nil { - let r: f64 = msg_send![color, redComponent]; - let g: f64 = msg_send![color, greenComponent]; - let b: f64 = msg_send![color, blueComponent]; - let a: f64 = msg_send![color, alphaComponent]; - hcursor += (r + g + b + a) * (255 << i) as f64; - } - } - Ok((c, hcursor as _)) - } -} - pub fn get_cursor_data(hcursor: u64) -> ResultType { autoreleasepool(|| unsafe_get_cursor_data(hcursor)) } @@ -631,9 +586,11 @@ pub fn get_cursor_data(hcursor: u64) -> ResultType { // https://github.com/stweil/OSXvnc/blob/master/OSXvnc-server/mousecursor.c fn unsafe_get_cursor_data(hcursor: u64) -> ResultType { unsafe { - let (c, hcursor2) = get_cursor_id()?; - if hcursor != hcursor2 { - bail!("cursor changed"); + // The cursor shown is captured even if it changed after get_cursor read the seed: the + // content names the shape, and that change moved the seed on, so the next poll captures it. + let c: id = msg_send![class!(NSCursor), currentSystemCursor]; + if c == nil { + bail!("Failed to call [NSCursor currentSystemCursor]"); } let hotspot: NSPoint = msg_send![c, hotSpot]; let img: id = msg_send![c, image]; diff --git a/src/platform/mod.rs b/src/platform/mod.rs index 7a7ddc035..2e6061664 100644 --- a/src/platform/mod.rs +++ b/src/platform/mod.rs @@ -228,6 +228,14 @@ mod tests { #[cfg(target_os = "macos")] macos::is_process_trusted(false); } + // A macOS capture takes the cursor shown, whatever seed it is asked for. + #[cfg(target_os = "macos")] + { + macos::reset_input_cache(); + if let Some(change) = get_cursor().unwrap() { + assert!(get_cursor_data(change.wrapping_add(1)).is_ok()); + } + } } #[test] fn test_get_cursor_pos() { diff --git a/src/server/connection.rs b/src/server/connection.rs index 03a7665a3..7f318326e 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -4001,6 +4001,17 @@ impl Connection { self.send(msg_out).await; } } + // Only to a connection the cursor service would send the shape to. + #[cfg(not(any(target_os = "android", target_os = "ios")))] + Some(misc::Union::RequestCursorData(id)) => { + if self.is_remote() + && (self.peer_keyboard_enabled() || self.show_remote_cursor) + { + if let Some(msg) = input_service::cursor_data_message(id) { + self.send((*msg).clone()).await; + } + } + } _ => {} }, Some(message::Union::AudioFrame(frame)) => { diff --git a/src/server/input_service.rs b/src/server/input_service.rs index 4eb6c7b76..3bf1ac664 100644 --- a/src/server/input_service.rs +++ b/src/server/input_service.rs @@ -42,6 +42,7 @@ struct StateCursor { impl super::service::Reset for StateCursor { fn reset(&mut self) { *self = Default::default(); + CURSOR_SHAPES.lock().unwrap().clear(); crate::platform::reset_input_cache(); fix_key_down_timeout(true); } @@ -406,7 +407,7 @@ fn run_cursor(sp: MouseCursorService, state: &mut StateCursor) -> ResultType<()> super::log::trace!("Cursor data cached, hcursor: {}", hcursor); msg = cached.clone(); } else { - let mut data = crate::get_cursor_data(hcursor)?; + let data = crate::get_cursor_data(hcursor)?; // File the shape under the id ACTUALLY served, not the one requested. Deliberately a // NEW name rather than shadowing `hcursor`: the insert below reads as the requested // id everywhere else in this function, and a cfg-gated shadow would make the two @@ -421,10 +422,7 @@ fn run_cursor(sp: MouseCursorService, state: &mut StateCursor) -> ResultType<()> let cache_key = served_id; #[cfg(not(all(target_os = "linux", feature = "drm")))] let cache_key = hcursor; - data.colors = hbb_common::compress::compress(&data.colors[..]).into(); - let mut tmp = Message::new(); - tmp.set_cursor_data(data); - msg = Arc::new(tmp); + msg = cursor_shape_message(data, hbb_common::compress::compress); // A DRM cursor id is derived from the shape's pixels plus geometry, so an animated // pointer mints a new id on every shape change and this map would grow for the life // of the service, each entry pinning a compressed cursor message. (Upstream's X11 @@ -438,8 +436,15 @@ fn run_cursor(sp: MouseCursorService, state: &mut StateCursor) -> ResultType<()> const CURSOR_CACHE_MAX: usize = 64; if state.cached_cursor_data.len() >= CURSOR_CACHE_MAX { state.cached_cursor_data.clear(); + CURSOR_SHAPES.lock().unwrap().clear(); + // The shape being sent was kept above; keep it past the clear. + shared_cursor_shape(msg.clone()); } } + limit_cursor_handles(&mut state.cached_cursor_data, &msg); + // A macOS seed marks a change, never the same one twice: filing it would only + // fill the map. + #[cfg(not(target_os = "macos"))] state.cached_cursor_data.insert(cache_key, msg.clone()); super::log::trace!("Cursor data updated, hcursor: {}", cache_key); } @@ -451,8 +456,12 @@ fn run_cursor(sp: MouseCursorService, state: &mut StateCursor) -> ResultType<()> { state.hcursor = drm_served_id; } - sp.send_shared(msg.clone()); - state.cursor_data = msg; + // A new handle or seed for the shape already shown changes nothing a controller sees; + // a shape is one message however many handles name it. + if !Arc::ptr_eq(&msg, &state.cursor_data) { + sp.send_shared(msg.clone()); + state.cursor_data = msg; + } } } sp.snapshot(|sps| { @@ -499,6 +508,95 @@ lazy_static::lazy_static! { static ref RELATIVE_MOUSE_CONNS: Arc>> = Default::default(); } +lazy_static::lazy_static! { + // Every shape the service has sent, by content id, for a controller that asks for one + // again. A shape under several handles is one message here, shared with `cached_cursor_data`, + // and cleared with it; see `limit_cursor_handles` for the ceilings. A shape dropped is + // rebuilt here the next time it is shown, in `run_cursor`, before any connection sends its + // `cursor_id`: whatever a controller has just been told to show, it can ask for. + static ref CURSOR_SHAPES: Mutex = Default::default(); +} + +/// The shapes sent, by content id, and the bytes they hold compressed. +#[derive(Default)] +struct CursorShapes { + shapes: HashMap>, + bytes: usize, +} + +impl CursorShapes { + fn get(&self, id: &u64) -> Option<&Arc> { + self.shapes.get(id) + } + + fn clear(&mut self) { + self.shapes.clear(); + self.bytes = 0; + } + + /// The message kept for this shape, `msg` if there was none. + fn keep(&mut self, cd: &CursorData, msg: &Arc) -> Arc { + let bytes = &mut self.bytes; + self.shapes + .entry(cd.id) + .or_insert_with(|| { + *bytes += cd.colors.len(); + msg.clone() + }) + .clone() + } +} + +/// The message for a shape the platform gave, named by content, so the per-connection send sends +/// a shape once however many handles the platform gives it; see +/// `is_peer_naming_cursors_by_content`. A shape sent before, under any handle, is reused without +/// being compressed again. +fn cursor_shape_message( + mut data: CursorData, + compress: impl FnOnce(&[u8]) -> Vec, +) -> Arc { + data.id = crate::cursor_content_id(data.width, data.height, data.hotx, data.hoty, &data.colors); + if let Some(msg) = cursor_data_message(data.id) { + return msg; + } + data.colors = compress(&data.colors[..]).into(); + let mut msg = Message::new(); + msg.set_cursor_data(data); + shared_cursor_shape(Arc::new(msg)) +} + +// A platform may mint a new handle each time it shows a shape (Chrome, Electron), so the handles +// filed would grow for the life of the service, each entry a u64 and an Arc even when every one +// names the same shape. Past this many handles, or this many bytes of compressed shapes, however +// few, the handles and the shapes start over together: a handle shown again is captured and named +// again. macOS files no handles, so there the bytes alone bound the shapes. +const CURSOR_HANDLES_MAX: usize = 4096; +const CURSOR_SHAPES_BYTES_MAX: usize = 32 << 20; + +fn limit_cursor_handles(handles: &mut HashMap>, sending: &Arc) { + if handles.len() >= CURSOR_HANDLES_MAX + || CURSOR_SHAPES.lock().unwrap().bytes >= CURSOR_SHAPES_BYTES_MAX + { + handles.clear(); + CURSOR_SHAPES.lock().unwrap().clear(); + // The shape being sent was kept; keep it past the clear. + shared_cursor_shape(sending.clone()); + } +} + +/// The message already kept for this shape if there is one, so every handle for it shares it. +fn shared_cursor_shape(msg: Arc) -> Arc { + let Some(message::Union::CursorData(cd)) = &msg.union else { + return msg; + }; + CURSOR_SHAPES.lock().unwrap().keep(cd, &msg) +} + +/// The CursorData message of a shape the service has sent, for `Misc::request_cursor_data`. +pub fn cursor_data_message(id: u64) -> Option> { + CURSOR_SHAPES.lock().unwrap().get(&id).cloned() +} + #[cfg(target_os = "linux")] lazy_static::lazy_static! { static ref WAYLAND_CLIPBOARD_INPUT_RECORDS: Arc>> = @@ -2533,3 +2631,96 @@ lazy_static::lazy_static! { (ControlKey::Delete, true), ].iter().map(|(a, b)| (a.value(), b.clone())).collect(); } + +#[cfg(test)] +mod cursor_shape_tests { + use super::*; + + fn shape(id: u64) -> Arc { + let mut msg = Message::new(); + msg.set_cursor_data(CursorData { + id, + ..Default::default() + }); + Arc::new(msg) + } + + // One test, since the shapes are kept process-wide. + #[test] + fn a_shape_is_kept_once_and_can_be_asked_for_again() { + let first = shared_cursor_shape(shape(u64::MAX - 1)); + let again = shared_cursor_shape(shape(u64::MAX - 1)); + assert!(Arc::ptr_eq(&first, &again), "a second handle shares it"); + let asked = cursor_data_message(u64::MAX - 1).expect("a sent shape is kept"); + assert!(Arc::ptr_eq(&asked, &first)); + assert!(cursor_data_message(u64::MAX - 2).is_none()); + + let raw = |handle| CursorData { + id: handle, + width: 4, + height: 4, + colors: vec![9u8; 4 * 4 * 4].into(), + ..Default::default() + }; + let mut compressed = 0; + let mut compress = |rgba: &[u8]| { + compressed += 1; + hbb_common::compress::compress(rgba) + }; + let shown = cursor_shape_message(raw(1), &mut compress); + let again = cursor_shape_message(raw(2), &mut compress); + assert!(Arc::ptr_eq(&shown, &again), "a new handle, the same shape"); + assert_eq!(compressed, 1, "a shape sent before is not compressed again"); + + let sending = shared_cursor_shape(shape(u64::MAX - 3)); + let mut few = HashMap::from([(1, shown.clone())]); + limit_cursor_handles(&mut few, &sending); + assert_eq!(few.len(), 1, "below the ceiling nothing goes"); + + let mut handles: HashMap> = (0..CURSOR_HANDLES_MAX as u64) + .map(|handle| (handle, shown.clone())) + .collect(); + limit_cursor_handles(&mut handles, &sending); + assert!( + handles.is_empty(), + "past the ceiling the handles start over" + ); + let Some(message::Union::CursorData(cd)) = &shown.union else { + panic!("a cursor shape"); + }; + assert!( + cursor_data_message(cd.id).is_none(), + "and the shapes with them" + ); + assert!( + cursor_data_message(u64::MAX - 3).is_some(), + "but the shape being sent is kept" + ); + + let big = |id| { + let mut msg = Message::new(); + msg.set_cursor_data(CursorData { + id, + colors: vec![0u8; CURSOR_SHAPES_BYTES_MAX / 2 + 1].into(), + ..Default::default() + }); + Arc::new(msg) + }; + shared_cursor_shape(big(u64::MAX - 4)); + let sending = shared_cursor_shape(big(u64::MAX - 5)); + let mut few = HashMap::from([(1, shown.clone())]); + limit_cursor_handles(&mut few, &sending); + assert!( + few.is_empty(), + "past the byte ceiling the handles start over" + ); + assert!(cursor_data_message(u64::MAX - 4).is_none()); + assert!(cursor_data_message(u64::MAX - 5).is_some()); + + super::super::service::Reset::reset(&mut StateCursor::default()); + assert!( + cursor_data_message(u64::MAX - 1).is_none(), + "a reset forgets them" + ); + } +} diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 2fa0ee6a5..f28752e77 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -76,6 +76,10 @@ pub struct Session { pub reconnect_count: Arc, pub last_audit_note: Arc>, pub audit_guid: Arc>, + // Every cursor shape as the peer sent it, compressed, by the id the UI knows it by. The + // peer sends a shape once, and a window asks for it whenever it has to draw it again. + #[cfg(feature = "flutter")] + pub cursor_shapes: Arc>>, } #[derive(Clone)]