From 3d1d037e1a9b3585f727bc16fb6753bbabe5e086 Mon Sep 17 00:00:00 2001 From: Haitao Pan Date: Thu, 4 Jun 2026 09:12:08 +0800 Subject: [PATCH] fix: WebRTC remote desktop connection, cleanup local fallback, and ignore .gradle cache --- .gitignore | 3 + lib/app/app_controller_desktop_gateway.dart | 5 - ...app_controller_desktop_thread_actions.dart | 5 - .../assistant_page_composer_skill_picker.dart | 20 +++ ...assistant_page_composer_state_helpers.dart | 1 + lib/features/desktop/desktop_client.dart | 71 ++++++-- .../desktop/desktop_input_handler.dart | 158 +++++++++++++----- lib/features/desktop/desktop_view.dart | 157 ++++++++++++----- .../settings/settings_logs_panel.dart | 1 + lib/runtime/gateway_acp_client.dart | 6 + lib/runtime/gateway_runtime_api.dart | 19 ++- .../gateway_runtime_session_client.dart | 1 + lib/runtime/runtime_controllers_entities.dart | 3 - macos/Runner/DebugProfile.entitlements | 2 + macos/Runner/Release.entitlements | 2 + .../assistant/assistant_lower_pane_test.dart | 34 ++++ .../features/desktop/desktop_client_test.dart | 46 +++++ .../desktop/desktop_input_handler_test.dart | 76 +++++++++ .../assistant_execution_target_test.dart | 132 +++++++++++++++ test/runtime/bridge_runtime_cleanup_test.dart | 1 + .../gateway_runtime_bridge_skills_test.dart | 120 +++++++++---- 21 files changed, 711 insertions(+), 152 deletions(-) create mode 100644 test/features/desktop/desktop_client_test.dart create mode 100644 test/features/desktop/desktop_input_handler_test.dart diff --git a/.gitignore b/.gitignore index 7e91d42b..b7918e9c 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,6 @@ app.*.map.json /rust/Cargo.lock /macos/Frameworks/*.dylib /macos/Frameworks/*.a + +# Gradle artifacts (including third_party) +**/.gradle/ diff --git a/lib/app/app_controller_desktop_gateway.dart b/lib/app/app_controller_desktop_gateway.dart index 13527efe..940cfe7d 100644 --- a/lib/app/app_controller_desktop_gateway.dart +++ b/lib/app/app_controller_desktop_gateway.dart @@ -232,11 +232,6 @@ extension AppControllerDesktopGateway on AppController { await refreshGatewayHealth(); await refreshAgents(); await refreshSessions(); - await skillsControllerInternal.refresh( - agentId: agentsControllerInternal.selectedAgentId.isEmpty - ? null - : agentsControllerInternal.selectedAgentId, - ); await modelsControllerInternal.refresh(); await cronJobsControllerInternal.refresh(); await devicesControllerInternal.refresh(quiet: true); diff --git a/lib/app/app_controller_desktop_thread_actions.dart b/lib/app/app_controller_desktop_thread_actions.dart index 1c13b671..7875a40f 100644 --- a/lib/app/app_controller_desktop_thread_actions.dart +++ b/lib/app/app_controller_desktop_thread_actions.dart @@ -179,11 +179,6 @@ extension AppControllerDesktopThreadActions on AppController { if (isAppOwnedAssistantSessionKeyInternal(sessionKey)) { await chatControllerInternal.loadSession(sessionKey); } - await skillsControllerInternal.refresh( - agentId: agentsControllerInternal.selectedAgentId.isEmpty - ? null - : agentsControllerInternal.selectedAgentId, - ); recomputeTasksInternal(); } diff --git a/lib/features/assistant/assistant_page_composer_skill_picker.dart b/lib/features/assistant/assistant_page_composer_skill_picker.dart index 74ae5af4..c3e77241 100644 --- a/lib/features/assistant/assistant_page_composer_skill_picker.dart +++ b/lib/features/assistant/assistant_page_composer_skill_picker.dart @@ -75,6 +75,7 @@ class SkillPickerPopoverInternal extends StatelessWidget { required this.selectedSkillKeys, required this.filteredSkills, required this.isLoading, + required this.errorText, required this.hasQuery, required this.onQueryChanged, required this.onToggleSkill, @@ -86,6 +87,7 @@ class SkillPickerPopoverInternal extends StatelessWidget { final List selectedSkillKeys; final List filteredSkills; final bool isLoading; + final String? errorText; final bool hasQuery; final ValueChanged onQueryChanged; final ValueChanged onToggleSkill; @@ -95,6 +97,7 @@ class SkillPickerPopoverInternal extends StatelessWidget { final palette = context.palette; final theme = Theme.of(context); final groupedSkills = skillPickerSectionsInternal(filteredSkills); + final hasError = !isLoading && (errorText?.trim().isNotEmpty ?? false); return Material( key: const Key('assistant-skill-picker-popover'), color: Colors.transparent, @@ -160,6 +163,11 @@ class SkillPickerPopoverInternal extends StatelessWidget { Text( isLoading ? appText('正在加载技能…', 'Loading skills…') + : hasError + ? appText( + '技能列表加载失败，请稍后重试。', + 'Could not load skills. Please try again.', + ) : hasQuery ? appText('没有匹配的技能。', 'No matching skills.') : appText( @@ -171,6 +179,18 @@ class SkillPickerPopoverInternal extends StatelessWidget { color: palette.textSecondary, ), ), + if (hasError) ...[ + const SizedBox(height: 8), + Text( + errorText!.trim(), + textAlign: TextAlign.center, + maxLines: 3, + overflow: TextOverflow.ellipsis, + style: theme.textTheme.bodySmall?.copyWith( + color: palette.textMuted, + ), + ), + ], ], ), ), diff --git a/lib/features/assistant/assistant_page_composer_state_helpers.dart b/lib/features/assistant/assistant_page_composer_state_helpers.dart index c437e1e5..5a5cc18b 100644 --- a/lib/features/assistant/assistant_page_composer_state_helpers.dart +++ b/lib/features/assistant/assistant_page_composer_state_helpers.dart @@ -95,6 +95,7 @@ Widget buildSkillPickerOverlayForInternal( selectedSkillKeys: state.widget.selectedSkillKeys, filteredSkills: state.filteredSkillOptionsInternal(), isLoading: state.widget.controller.skillsController.loading, + errorText: state.widget.controller.skillsController.error, hasQuery: state.skillPickerQueryInternal.trim().isNotEmpty, onQueryChanged: state.setSkillPickerQueryInternal, onToggleSkill: (skillKey) => state.widget.onToggleSkill(skillKey), diff --git a/lib/features/desktop/desktop_client.dart b/lib/features/desktop/desktop_client.dart index e66640b0..3ae097e5 100644 --- a/lib/features/desktop/desktop_client.dart +++ b/lib/features/desktop/desktop_client.dart @@ -3,6 +3,33 @@ import 'dart:convert'; import 'package:flutter_webrtc/flutter_webrtc.dart'; import '../../app/app_controller.dart'; +String desktopConnectionStateName(RTCPeerConnectionState state) { + final value = state.toString().split('.').last; + return value.replaceFirst('RTCPeerConnectionState', '').toLowerCase(); +} + +Map desktopOfferParams({ + required String sessionId, + required String? sdpOffer, + required String display, + required int width, + required int height, + required int fps, + required int bitrate, + required bool useGpu, +}) { + return { + 'sessionId': sessionId, + 'sdpOffer': sdpOffer, + 'display': display, + 'width': width, + 'height': height, + 'fps': fps, + 'bitrate': bitrate, + 'useGpu': useGpu, + }; +} + class DesktopClient { DesktopClient({required this.controller, required this.sessionId}); @@ -42,7 +69,7 @@ class DesktopClient { try { final config = { 'iceServers': [ - {'url': 'stun:stun.l.google.com:19302'} + {'urls': 'stun:stun.l.google.com:19302'}, ], 'sdpSemantics': 'unified-plan', }; @@ -60,18 +87,27 @@ class DesktopClient { }; _peerConnection!.onConnectionState = (state) { - _stateController.add(state.toString().split('.').last); + _stateController.add(desktopConnectionStateName(state)); }; // Create data channel for inputs BEFORE creating offer final dcConfig = RTCDataChannelInit()..ordered = true; - _dataChannel = - await _peerConnection!.createDataChannel('input', dcConfig); + _dataChannel = await _peerConnection!.createDataChannel( + 'input', + dcConfig, + ); // Handle ICE Candidates generated locally + final List iceQueue = []; + bool isRemoteSet = false; + _peerConnection!.onIceCandidate = (RTCIceCandidate? candidate) { if (candidate != null) { - _sendIceCandidate(candidate); + if (isRemoteSet) { + unawaited(_sendIceCandidate(candidate)); + } else { + iceQueue.add(candidate); + } } }; @@ -88,16 +124,16 @@ class DesktopClient { // Send SDP Offer to Bridge final response = await controller.gatewayAcpClientInternal.request( method: 'xworkmate.desktop.offer', - params: { - 'sessionId': sessionId, - 'sdpOffer': offer.sdp, - 'display': display, - 'width': width.toString(), - 'height': height.toString(), - 'fps': fps.toString(), - 'bitrate': bitrate.toString(), - 'useGpu': useGpu.toString(), - }, + params: desktopOfferParams( + sessionId: sessionId, + sdpOffer: offer.sdp, + display: display, + width: width, + height: height, + fps: fps, + bitrate: bitrate, + useGpu: useGpu, + ), ); final sdpAnswerData = response['result']?['sdpAnswer']; @@ -116,6 +152,11 @@ class DesktopClient { answer = RTCSessionDescription(sdpAnswerData.toString(), 'answer'); } await _peerConnection!.setRemoteDescription(answer); + isRemoteSet = true; + for (final candidate in iceQueue) { + unawaited(_sendIceCandidate(candidate)); + } + iceQueue.clear(); _isConnecting = false; } catch (e) { diff --git a/lib/features/desktop/desktop_input_handler.dart b/lib/features/desktop/desktop_input_handler.dart index 07cfbbeb..bf1bbacf 100644 --- a/lib/features/desktop/desktop_input_handler.dart +++ b/lib/features/desktop/desktop_input_handler.dart @@ -8,43 +8,39 @@ class DesktopInputHandler { final void Function(Map event) onSendInput; int _lastPressedButton = 1; // Default to left click - void handlePointerMove(PointerEvent event, Size widgetSize) { - if (widgetSize.width == 0 || widgetSize.height == 0) return; - - final x = event.localPosition.dx / widgetSize.width; - final y = event.localPosition.dy / widgetSize.height; - - final cx = x.clamp(0.0, 1.0); - final cy = y.clamp(0.0, 1.0); + void handlePointerMove( + PointerEvent event, + Size widgetSize, { + Size? contentSize, + }) { + final position = desktopContentPosition( + event.localPosition, + widgetSize, + contentSize: contentSize, + ); + if (position == null) return; - onSendInput({ - 'type': 'mouse_move', - 'x': cx, - 'y': cy, - }); + onSendInput({'type': 'mouse_move', 'x': position.dx, 'y': position.dy}); } - void handlePointerDown(PointerDownEvent event, Size widgetSize) { - if (widgetSize.width == 0 || widgetSize.height == 0) return; - - final x = event.localPosition.dx / widgetSize.width; - final y = event.localPosition.dy / widgetSize.height; - final cx = x.clamp(0.0, 1.0); - final cy = y.clamp(0.0, 1.0); + void handlePointerDown( + PointerDownEvent event, + Size widgetSize, { + Size? contentSize, + }) { + final position = desktopContentPosition( + event.localPosition, + widgetSize, + contentSize: contentSize, + ); + if (position == null) return; // Send move event first to ensure click hits the exact coordinates - onSendInput({ - 'type': 'mouse_move', - 'x': cx, - 'y': cy, - }); + onSendInput({'type': 'mouse_move', 'x': position.dx, 'y': position.dy}); _lastPressedButton = _mapPointerButtons(event.buttons); - onSendInput({ - 'type': 'mouse_down', - 'button': _lastPressedButton, - }); + onSendInput({'type': 'mouse_down', 'button': _lastPressedButton}); } void handlePointerUp(PointerUpEvent event, Size widgetSize) { @@ -55,29 +51,21 @@ class DesktopInputHandler { releasedButton = _mapPointerButtons(event.buttons); } - onSendInput({ - 'type': 'mouse_up', - 'button': releasedButton, - }); + onSendInput({'type': 'mouse_up', 'button': releasedButton}); } void handleScroll(PointerScrollEvent event) { // 4 = scroll up, 5 = scroll down in X11/xdotool button maps final button = event.scrollDelta.dy < 0 ? 4 : 5; - onSendInput({ - 'type': 'scroll', - 'button': button, - }); + onSendInput({'type': 'scroll', 'button': button}); } void handleKeyEvent(KeyEvent event) { final isDown = event is KeyDownEvent || event is KeyRepeatEvent; - final keyLabel = event.logicalKey.keyLabel; + final keyLabel = desktopKeyName(event.logicalKey); + if (keyLabel == null) return; - onSendInput({ - 'type': isDown ? 'key_down' : 'key_up', - 'key': keyLabel, - }); + onSendInput({'type': isDown ? 'key_down' : 'key_up', 'key': keyLabel}); } int _mapPointerButtons(int buttons) { @@ -92,3 +80,89 @@ class DesktopInputHandler { return 1; } } + +String? desktopKeyName(LogicalKeyboardKey key) { + if (key == LogicalKeyboardKey.enter) return 'Return'; + if (key == LogicalKeyboardKey.numpadEnter) return 'Return'; + if (key == LogicalKeyboardKey.space) return 'space'; + if (key == LogicalKeyboardKey.backspace) return 'BackSpace'; + if (key == LogicalKeyboardKey.tab) return 'Tab'; + if (key == LogicalKeyboardKey.escape) return 'Escape'; + if (key == LogicalKeyboardKey.delete) return 'Delete'; + if (key == LogicalKeyboardKey.arrowLeft) return 'Left'; + if (key == LogicalKeyboardKey.arrowRight) return 'Right'; + if (key == LogicalKeyboardKey.arrowUp) return 'Up'; + if (key == LogicalKeyboardKey.arrowDown) return 'Down'; + if (key == LogicalKeyboardKey.home) return 'Home'; + if (key == LogicalKeyboardKey.end) return 'End'; + if (key == LogicalKeyboardKey.pageUp) return 'Page_Up'; + if (key == LogicalKeyboardKey.pageDown) return 'Page_Down'; + + final label = key.keyLabel; + if (label.isEmpty) return null; + if (label.length == 1) { + final punctuation = _xdotoolPunctuationNames[label]; + if (punctuation != null) return punctuation; + if (RegExp(r'^[A-Z]$').hasMatch(label)) { + return label.toLowerCase(); + } + } + return label; +} + +const Map _xdotoolPunctuationNames = { + '/': 'slash', + '.': 'period', + ',': 'comma', + '-': 'minus', + '_': 'underscore', + '=': 'equal', + ';': 'semicolon', + "'": 'apostrophe', + '`': 'grave', + '[': 'bracketleft', + ']': 'bracketright', + '\\': 'backslash', +}; + +Offset? desktopContentPosition( + Offset localPosition, + Size viewportSize, { + Size? contentSize, +}) { + if (viewportSize.width <= 0 || viewportSize.height <= 0) return null; + + final resolvedContentSize = contentSize; + if (resolvedContentSize == null || + resolvedContentSize.width <= 0 || + resolvedContentSize.height <= 0) { + return Offset( + (localPosition.dx / viewportSize.width).clamp(0.0, 1.0), + (localPosition.dy / viewportSize.height).clamp(0.0, 1.0), + ); + } + + final viewportAspect = viewportSize.width / viewportSize.height; + final contentAspect = resolvedContentSize.width / resolvedContentSize.height; + + double drawnWidth; + double drawnHeight; + double offsetX; + double offsetY; + if (viewportAspect > contentAspect) { + drawnHeight = viewportSize.height; + drawnWidth = drawnHeight * contentAspect; + offsetX = (viewportSize.width - drawnWidth) / 2; + offsetY = 0; + } else { + drawnWidth = viewportSize.width; + drawnHeight = drawnWidth / contentAspect; + offsetX = 0; + offsetY = (viewportSize.height - drawnHeight) / 2; + } + + return Offset( + ((localPosition.dx - offsetX) / drawnWidth).clamp(0.0, 1.0), + ((localPosition.dy - offsetY) / drawnHeight).clamp(0.0, 1.0), + ); +} diff --git a/lib/features/desktop/desktop_view.dart b/lib/features/desktop/desktop_view.dart index 79f41038..e487312d 100644 --- a/lib/features/desktop/desktop_view.dart +++ b/lib/features/desktop/desktop_view.dart @@ -27,20 +27,31 @@ class _DesktopViewState extends State { final RTCVideoRenderer _localRenderer = RTCVideoRenderer(); late DesktopClient _client; DesktopInputHandler? _inputHandler; - + // Settings controllers - final TextEditingController _displayController = TextEditingController(text: ':0.0'); - final TextEditingController _widthController = TextEditingController(text: '1280'); - final TextEditingController _heightController = TextEditingController(text: '720'); - final TextEditingController _fpsController = TextEditingController(text: '30'); - final TextEditingController _bitrateController = TextEditingController(text: '2000'); - + final TextEditingController _displayController = TextEditingController( + text: ':0.0', + ); + final TextEditingController _widthController = TextEditingController( + text: '1280', + ); + final TextEditingController _heightController = TextEditingController( + text: '720', + ); + final TextEditingController _fpsController = TextEditingController( + text: '30', + ); + final TextEditingController _bitrateController = TextEditingController( + text: '2000', + ); + bool _useGpu = false; bool _showAdvancedOptions = false; String _connectionState = 'disconnected'; bool _hasStream = false; bool _isFocused = false; - + Size _remoteDesktopSize = const Size(1280, 720); + final FocusNode _viewportFocusNode = FocusNode(); final GlobalKey _viewportKey = GlobalKey(); @@ -55,11 +66,13 @@ class _DesktopViewState extends State { controller: widget.controller, sessionId: 'remote-desktop-session', ); - _inputHandler = DesktopInputHandler(onSendInput: (event) { - if (_connectionState == 'connected') { - _client.sendInput(event); - } - }); + _inputHandler = DesktopInputHandler( + onSendInput: (event) { + if (_connectionState == 'connected') { + _client.sendInput(event); + } + }, + ); _streamSubscription = _client.onRemoteStream.listen((stream) { if (mounted) { @@ -74,7 +87,8 @@ class _DesktopViewState extends State { if (mounted) { setState(() { _connectionState = state.toLowerCase(); - if (_connectionState == 'disconnected' || _connectionState == 'failed') { + if (_connectionState == 'disconnected' || + _connectionState == 'failed') { _hasStream = false; _localRenderer.srcObject = null; } @@ -111,6 +125,7 @@ class _DesktopViewState extends State { final height = int.tryParse(_heightController.text) ?? 720; final fps = int.tryParse(_fpsController.text) ?? 30; final bitrate = int.tryParse(_bitrateController.text) ?? 2000; + _remoteDesktopSize = Size(width.toDouble(), height.toDouble()); try { await _client.connect( @@ -135,7 +150,8 @@ class _DesktopViewState extends State { } Size _getViewportSize() { - final renderBox = _viewportKey.currentContext?.findRenderObject() as RenderBox?; + final renderBox = + _viewportKey.currentContext?.findRenderObject() as RenderBox?; return renderBox?.size ?? Size.zero; } @@ -168,11 +184,16 @@ class _DesktopViewState extends State { backgroundColor: _connectionState == 'connected' ? Colors.redAccent : (_connectionState == 'connecting' - ? Colors.orangeAccent - : theme.colorScheme.primary), + ? Colors.orangeAccent + : theme.colorScheme.primary), foregroundColor: Colors.white, - padding: const EdgeInsets.symmetric(horizontal: 20, vertical: 16), - shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(8)), + padding: const EdgeInsets.symmetric( + horizontal: 20, + vertical: 16, + ), + shape: RoundedRectangleBorder( + borderRadius: BorderRadius.circular(8), + ), ), icon: Icon( _connectionState == 'connected' @@ -182,26 +203,31 @@ class _DesktopViewState extends State { label: Text( _connectionState == 'connected' ? '断开连接' - : (_connectionState == 'connecting' ? '正在连接...' : '连接桌面'), + : (_connectionState == 'connecting' + ? '正在连接...' + : '连接桌面'), style: const TextStyle(fontWeight: FontWeight.bold), ), ), // Status Indicator Container( - padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 6), + padding: const EdgeInsets.symmetric( + horizontal: 12, + vertical: 6, + ), decoration: BoxDecoration( color: _connectionState == 'connected' ? Colors.green.withValues(alpha: 0.15) : (_connectionState == 'connecting' - ? Colors.orange.withValues(alpha: 0.15) - : Colors.grey.withValues(alpha: 0.15)), + ? Colors.orange.withValues(alpha: 0.15) + : Colors.grey.withValues(alpha: 0.15)), borderRadius: BorderRadius.circular(16), border: Border.all( color: _connectionState == 'connected' ? Colors.green : (_connectionState == 'connecting' - ? Colors.orange - : Colors.grey), + ? Colors.orange + : Colors.grey), width: 1, ), ), @@ -215,8 +241,8 @@ class _DesktopViewState extends State { color: _connectionState == 'connected' ? Colors.green : (_connectionState == 'connecting' - ? Colors.orange - : Colors.grey), + ? Colors.orange + : Colors.grey), shape: BoxShape.circle, ), ), @@ -224,15 +250,19 @@ class _DesktopViewState extends State { Text( _connectionState == 'connected' ? '已连接' - : (_connectionState == 'connecting' ? '连接中' : '未连接'), + : (_connectionState == 'connecting' + ? '连接中' + : '未连接'), style: TextStyle( fontSize: 12, fontWeight: FontWeight.bold, color: _connectionState == 'connected' ? Colors.green : (_connectionState == 'connecting' - ? Colors.orange - : (isDark ? Colors.white70 : Colors.black87)), + ? Colors.orange + : (isDark + ? Colors.white70 + : Colors.black87)), ), ), ], @@ -240,15 +270,25 @@ class _DesktopViewState extends State { ), // Advanced Options Toggle TextButton.icon( - onPressed: () => setState(() => _showAdvancedOptions = !_showAdvancedOptions), - icon: Icon(_showAdvancedOptions ? Icons.expand_less : Icons.expand_more), + onPressed: () => setState( + () => _showAdvancedOptions = !_showAdvancedOptions, + ), + icon: Icon( + _showAdvancedOptions + ? Icons.expand_less + : Icons.expand_more, + ), label: const Text('高级选项'), ), // Maximize Toggle if (widget.onToggleMaximize != null) IconButton( onPressed: widget.onToggleMaximize, - icon: Icon(widget.isMaximized ? Icons.fullscreen_exit_rounded : Icons.fullscreen_rounded), + icon: Icon( + widget.isMaximized + ? Icons.fullscreen_exit_rounded + : Icons.fullscreen_rounded, + ), tooltip: widget.isMaximized ? '恢复默认大小' : '最大化', ), ], @@ -332,7 +372,7 @@ class _DesktopViewState extends State { ), ), ), - + const SizedBox(height: 16), // Stream Viewport Card @@ -345,7 +385,9 @@ class _DesktopViewState extends State { }); }, onKeyEvent: (node, event) { - if (_isFocused && _connectionState == 'connected' && _inputHandler != null) { + if (_isFocused && + _connectionState == 'connected' && + _inputHandler != null) { _inputHandler!.handleKeyEvent(event); return KeyEventResult.handled; } @@ -354,7 +396,9 @@ class _DesktopViewState extends State { child: Container( key: _viewportKey, decoration: BoxDecoration( - color: isDark ? Colors.black26 : Colors.black.withValues(alpha: 0.04), + color: isDark + ? Colors.black26 + : Colors.black.withValues(alpha: 0.04), borderRadius: BorderRadius.circular(12), border: Border.all( color: _isFocused @@ -373,12 +417,20 @@ class _DesktopViewState extends State { behavior: HitTestBehavior.opaque, onPointerHover: (event) { if (_inputHandler != null) { - _inputHandler!.handlePointerMove(event, _getViewportSize()); + _inputHandler!.handlePointerMove( + event, + _getViewportSize(), + contentSize: _remoteDesktopSize, + ); } }, onPointerMove: (event) { if (_inputHandler != null) { - _inputHandler!.handlePointerMove(event, _getViewportSize()); + _inputHandler!.handlePointerMove( + event, + _getViewportSize(), + contentSize: _remoteDesktopSize, + ); } }, onPointerDown: (event) { @@ -386,22 +438,31 @@ class _DesktopViewState extends State { _viewportFocusNode.requestFocus(); } if (_inputHandler != null) { - _inputHandler!.handlePointerDown(event, _getViewportSize()); + _inputHandler!.handlePointerDown( + event, + _getViewportSize(), + contentSize: _remoteDesktopSize, + ); } }, onPointerUp: (event) { if (_inputHandler != null) { - _inputHandler!.handlePointerUp(event, _getViewportSize()); + _inputHandler!.handlePointerUp( + event, + _getViewportSize(), + ); } }, onPointerSignal: (event) { - if (event is PointerScrollEvent && _inputHandler != null) { + if (event is PointerScrollEvent && + _inputHandler != null) { _inputHandler!.handleScroll(event); } }, child: RTCVideoView( _localRenderer, - objectFit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain, + objectFit: RTCVideoViewObjectFit + .RTCVideoViewObjectFitContain, ), ), ), @@ -418,7 +479,9 @@ class _DesktopViewState extends State { Icon( Icons.monitor_rounded, size: 64, - color: theme.colorScheme.onSurface.withValues(alpha: 0.2), + color: theme.colorScheme.onSurface.withValues( + alpha: 0.2, + ), ), const SizedBox(height: 16), Text( @@ -426,7 +489,8 @@ class _DesktopViewState extends State { ? '正在建立 WebRTC 连接，请稍候...' : '未开启远程桌面流。点击“连接桌面”启动视频流。', style: TextStyle( - color: theme.colorScheme.onSurface.withValues(alpha: 0.6), + color: theme.colorScheme.onSurface + .withValues(alpha: 0.6), fontSize: 14, ), ), @@ -449,7 +513,10 @@ class _DesktopViewState extends State { opacity: _isFocused ? 0.3 : 0.8, duration: const Duration(milliseconds: 200), child: Container( - padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4), + padding: const EdgeInsets.symmetric( + horizontal: 8, + vertical: 4, + ), decoration: BoxDecoration( color: Colors.black.withValues(alpha: 0.6), borderRadius: BorderRadius.circular(4), diff --git a/lib/features/settings/settings_logs_panel.dart b/lib/features/settings/settings_logs_panel.dart index 2bada5d1..2642d607 100644 --- a/lib/features/settings/settings_logs_panel.dart +++ b/lib/features/settings/settings_logs_panel.dart @@ -56,6 +56,7 @@ class _SettingsLogsPanelState extends State { } } } catch (e) { + appLog('Failed to fetch system status: $e'); if (mounted) { setState(() { _bridgeStatus = 'error'; diff --git a/lib/runtime/gateway_acp_client.dart b/lib/runtime/gateway_acp_client.dart index 6afe84d6..48027637 100644 --- a/lib/runtime/gateway_acp_client.dart +++ b/lib/runtime/gateway_acp_client.dart @@ -1411,6 +1411,7 @@ class GatewayAcpRuntimeSessionClient implements GatewayRuntimeSessionClient { required String method, Map? params, Duration timeout = const Duration(seconds: 15), + bool allowErrorPayload = false, }) async { final envelope = await client.request( method: 'xworkmate.gateway.request', @@ -1423,6 +1424,11 @@ class GatewayAcpRuntimeSessionClient implements GatewayRuntimeSessionClient { onNotification: _handleNotification, ); final result = client.asMap(envelope['result']); + if (allowErrorPayload && + !(client.boolValue(result['ok']) ?? false) && + result.containsKey('payload')) { + return result['payload']; + } _throwGatewayResultIfNeeded(result, '$method request failed'); return result['payload']; } diff --git a/lib/runtime/gateway_runtime_api.dart b/lib/runtime/gateway_runtime_api.dart index 8bf8eed2..6f074fdf 100644 --- a/lib/runtime/gateway_runtime_api.dart +++ b/lib/runtime/gateway_runtime_api.dart @@ -199,14 +199,19 @@ extension GatewayRuntimeApiInternal on GatewayRuntime { Future> _listSkillsInternal({ String? agentId, }) async { + final params = { + if (agentId != null && agentId.trim().isNotEmpty) + 'agentId': agentId.trim(), + }; final payload = asMap( - await request( - 'skills.status', - params: { - if (agentId != null && agentId.trim().isNotEmpty) - 'agentId': agentId.trim(), - }, - ), + sessionClientInternal == null + ? await request('skills.status', params: params) + : await sessionClientInternal!.request( + runtimeId: runtimeIdInternal, + method: 'skills.status', + params: params, + allowErrorPayload: true, + ), ); return asList(payload['skills']) .map((item) { diff --git a/lib/runtime/gateway_runtime_session_client.dart b/lib/runtime/gateway_runtime_session_client.dart index 11ff1fff..6ef2d119 100644 --- a/lib/runtime/gateway_runtime_session_client.dart +++ b/lib/runtime/gateway_runtime_session_client.dart @@ -186,6 +186,7 @@ abstract class GatewayRuntimeSessionClient { required String method, Map? params, Duration timeout = const Duration(seconds: 15), + bool allowErrorPayload = false, }); Future disconnect({required String runtimeId}); diff --git a/lib/runtime/runtime_controllers_entities.dart b/lib/runtime/runtime_controllers_entities.dart index 232e205b..9e146943 100644 --- a/lib/runtime/runtime_controllers_entities.dart +++ b/lib/runtime/runtime_controllers_entities.dart @@ -37,9 +37,6 @@ class SkillsController extends ChangeNotifier { errorInternal = null; notifyListeners(); try { - await runtimeInternal.ensureBridgeSessionConnected( - selectedAgentId: agentId?.trim() ?? '', - ); itemsInternal = await runtimeInternal.listSkills(agentId: agentId); } catch (error) { errorInternal = error.toString(); diff --git a/macos/Runner/DebugProfile.entitlements b/macos/Runner/DebugProfile.entitlements index b84f7313..9dbcc0f3 100644 --- a/macos/Runner/DebugProfile.entitlements +++ b/macos/Runner/DebugProfile.entitlements @@ -2,6 +2,8 @@ + com.apple.security.network.client + com.apple.security.files.bookmarks.app-scope com.apple.security.files.user-selected.read-only diff --git a/macos/Runner/Release.entitlements b/macos/Runner/Release.entitlements index 0791a622..21547fe5 100644 --- a/macos/Runner/Release.entitlements +++ b/macos/Runner/Release.entitlements @@ -2,6 +2,8 @@ + com.apple.security.network.client + com.apple.security.files.bookmarks.app-scope diff --git a/test/features/assistant/assistant_lower_pane_test.dart b/test/features/assistant/assistant_lower_pane_test.dart index 3b211c6b..032fad5a 100644 --- a/test/features/assistant/assistant_lower_pane_test.dart +++ b/test/features/assistant/assistant_lower_pane_test.dart @@ -635,6 +635,7 @@ void main() { ), ], isLoading: false, + errorText: null, hasQuery: false, onQueryChanged: (_) {}, onToggleSkill: toggledKeys.add, @@ -684,6 +685,7 @@ void main() { ), ], isLoading: false, + errorText: null, hasQuery: true, onQueryChanged: (_) {}, onToggleSkill: (_) {}, @@ -697,6 +699,38 @@ void main() { expect(find.text('Workspace Skills'), findsNothing); expect(find.text('Agent Skills'), findsNothing); }); + + testWidgets( + 'empty skill picker shows refresh error instead of empty state', + (tester) async { + final searchController = TextEditingController(); + final focusNode = FocusNode(); + addTearDown(searchController.dispose); + addTearDown(focusNode.dispose); + + await tester.pumpWidget( + _buildTestApp( + child: SkillPickerPopoverInternal( + maxHeight: 360, + searchController: searchController, + searchFocusNode: focusNode, + selectedSkillKeys: const [], + filteredSkills: const [], + isLoading: false, + errorText: 'skills.status request failed', + hasQuery: false, + onQueryChanged: (_) {}, + onToggleSkill: (_) {}, + ), + ), + ); + await tester.pumpAndSettle(); + + expect(find.text('技能列表加载失败，请稍后重试。'), findsOneWidget); + expect(find.text('skills.status request failed'), findsOneWidget); + expect(find.text('当前没有已加载技能。'), findsNothing); + }, + ); }); } diff --git a/test/features/desktop/desktop_client_test.dart b/test/features/desktop/desktop_client_test.dart new file mode 100644 index 00000000..40f6722c --- /dev/null +++ b/test/features/desktop/desktop_client_test.dart @@ -0,0 +1,46 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter_webrtc/flutter_webrtc.dart'; +import 'package:xworkmate/features/desktop/desktop_client.dart'; + +void main() { + group('DesktopClient protocol helpers', () { + test('normalizes WebRTC connection states for view gating', () { + expect( + desktopConnectionStateName( + RTCPeerConnectionState.RTCPeerConnectionStateConnected, + ), + 'connected', + ); + expect( + desktopConnectionStateName( + RTCPeerConnectionState.RTCPeerConnectionStateFailed, + ), + 'failed', + ); + }); + + test('builds desktop offer params with native numeric values', () { + final params = desktopOfferParams( + sessionId: 'desktop-session-1', + sdpOffer: 'v=0', + display: ':0.0', + width: 1280, + height: 720, + fps: 30, + bitrate: 2000, + useGpu: false, + ); + + expect(params['sessionId'], 'desktop-session-1'); + expect(params['sdpOffer'], 'v=0'); + expect(params['display'], ':0.0'); + expect(params['width'], isA()); + expect(params['height'], isA()); + expect(params['fps'], isA()); + expect(params['bitrate'], isA()); + expect(params['useGpu'], isA()); + expect(params['width'], 1280); + expect(params['height'], 720); + }); + }); +} diff --git a/test/features/desktop/desktop_input_handler_test.dart b/test/features/desktop/desktop_input_handler_test.dart new file mode 100644 index 00000000..99a798e2 --- /dev/null +++ b/test/features/desktop/desktop_input_handler_test.dart @@ -0,0 +1,76 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter/services.dart'; +import 'package:xworkmate/features/desktop/desktop_input_handler.dart'; + +void main() { + group('desktopKeyName', () { + test('uses xdotool-compatible names for basic terminal keys', () { + expect(desktopKeyName(LogicalKeyboardKey.enter), 'Return'); + expect(desktopKeyName(LogicalKeyboardKey.numpadEnter), 'Return'); + expect(desktopKeyName(LogicalKeyboardKey.space), 'space'); + expect(desktopKeyName(LogicalKeyboardKey.backspace), 'BackSpace'); + expect(desktopKeyName(LogicalKeyboardKey.tab), 'Tab'); + expect(desktopKeyName(LogicalKeyboardKey.escape), 'Escape'); + }); + + test('uses xdotool-compatible names for navigation keys', () { + expect(desktopKeyName(LogicalKeyboardKey.arrowLeft), 'Left'); + expect(desktopKeyName(LogicalKeyboardKey.arrowRight), 'Right'); + expect(desktopKeyName(LogicalKeyboardKey.arrowUp), 'Up'); + expect(desktopKeyName(LogicalKeyboardKey.arrowDown), 'Down'); + expect(desktopKeyName(LogicalKeyboardKey.pageUp), 'Page_Up'); + expect(desktopKeyName(LogicalKeyboardKey.pageDown), 'Page_Down'); + }); + + test('keeps printable keys available for shell and browser text input', () { + expect(desktopKeyName(LogicalKeyboardKey.keyA), 'a'); + expect(desktopKeyName(LogicalKeyboardKey.digit1), '1'); + expect(desktopKeyName(LogicalKeyboardKey.period), 'period'); + expect(desktopKeyName(LogicalKeyboardKey.slash), 'slash'); + expect(desktopKeyName(LogicalKeyboardKey.minus), 'minus'); + }); + }); + + group('desktopContentPosition', () { + test('maps directly when viewport and content share an aspect ratio', () { + final position = desktopContentPosition( + const Offset(640, 360), + const Size(1280, 720), + contentSize: const Size(1280, 720), + ); + + expect(position, const Offset(0.5, 0.5)); + }); + + test('subtracts horizontal letterbox before normalizing pointer input', () { + final position = desktopContentPosition( + const Offset(160, 360), + const Size(1600, 720), + contentSize: const Size(1280, 720), + ); + + expect(position!.dx, closeTo(0.0, 0.001)); + expect(position.dy, closeTo(0.5, 0.001)); + }); + + test('subtracts vertical letterbox before normalizing pointer input', () { + final position = desktopContentPosition( + const Offset(640, 140), + const Size(1280, 1000), + contentSize: const Size(1280, 720), + ); + + expect(position!.dx, closeTo(0.5, 0.001)); + expect(position.dy, closeTo(0.0, 0.001)); + }); + + test('keeps legacy full-viewport mapping when content size is unknown', () { + final position = desktopContentPosition( + const Offset(640, 360), + const Size(1280, 720), + ); + + expect(position, const Offset(0.5, 0.5)); + }); + }); +} diff --git a/test/runtime/assistant_execution_target_test.dart b/test/runtime/assistant_execution_target_test.dart index 0f4fd10c..cd443ec2 100644 --- a/test/runtime/assistant_execution_target_test.dart +++ b/test/runtime/assistant_execution_target_test.dart @@ -659,6 +659,138 @@ void main() { }, ); + test( + 'skill picker selection allows multiple skills and independent deselect', + () async { + final controller = _connectedGatewayController( + _RecordingGoTaskServiceClient(), + ); + addTearDown(controller.dispose); + controller.skillsControllerInternal.itemsInternal = + const [ + GatewaySkillSummary( + name: 'PDF Writer', + description: 'Write PDF documents', + source: 'openclaw-workspace', + skillKey: 'pdf', + primaryEnv: null, + eligible: true, + disabled: false, + missingBins: [], + missingEnv: [], + missingConfig: [], + ), + GatewaySkillSummary( + name: 'Browser Automation', + description: 'Use browser automation', + source: 'agents-skills-personal', + skillKey: 'browser-automation', + primaryEnv: null, + eligible: true, + disabled: false, + missingBins: [], + missingEnv: [], + missingConfig: [], + ), + ]; + await _selectGatewaySession(controller, 'unit-skill-multi-select-task'); + + await controller.toggleAssistantSkillForSession( + 'unit-skill-multi-select-task', + 'pdf', + ); + await controller.toggleAssistantSkillForSession( + 'unit-skill-multi-select-task', + 'browser-automation', + ); + + expect( + controller.assistantSelectedSkillKeysForSession( + 'unit-skill-multi-select-task', + ), + const ['pdf', 'browser-automation'], + ); + + await controller.toggleAssistantSkillForSession( + 'unit-skill-multi-select-task', + 'browser-automation', + ); + + expect( + controller.assistantSelectedSkillKeysForSession( + 'unit-skill-multi-select-task', + ), + const ['pdf'], + ); + }, + ); + + test('skill selection is isolated across task sessions', () async { + final controller = _connectedGatewayController( + _RecordingGoTaskServiceClient(), + ); + addTearDown(controller.dispose); + controller.skillsControllerInternal.itemsInternal = + const [ + GatewaySkillSummary( + name: 'PDF Writer', + description: 'Write PDF documents', + source: 'openclaw-workspace', + skillKey: 'pdf', + primaryEnv: null, + eligible: true, + disabled: false, + missingBins: [], + missingEnv: [], + missingConfig: [], + ), + GatewaySkillSummary( + name: 'Browser Automation', + description: 'Use browser automation', + source: 'agents-skills-personal', + skillKey: 'browser-automation', + primaryEnv: null, + eligible: true, + disabled: false, + missingBins: [], + missingEnv: [], + missingConfig: [], + ), + ]; + + await _selectGatewaySession(controller, 'unit-skill-isolated-a'); + await controller.toggleAssistantSkillForSession( + 'unit-skill-isolated-a', + 'pdf', + ); + await _selectGatewaySession(controller, 'unit-skill-isolated-b'); + + expect( + controller.assistantSelectedSkillKeysForSession( + 'unit-skill-isolated-b', + ), + isEmpty, + ); + + await controller.toggleAssistantSkillForSession( + 'unit-skill-isolated-b', + 'browser-automation', + ); + + expect( + controller + .buildExternalAcpRoutingForSessionInternal('unit-skill-isolated-a') + .explicitSkills, + const ['pdf'], + ); + expect( + controller + .buildExternalAcpRoutingForSessionInternal('unit-skill-isolated-b') + .explicitSkills, + const ['browser-automation'], + ); + }); + test('skill selection ignores stale non-bridge skill keys', () { final controller = AppController( environmentOverride: const {}, diff --git a/test/runtime/bridge_runtime_cleanup_test.dart b/test/runtime/bridge_runtime_cleanup_test.dart index 4562c9e6..4f87d05d 100644 --- a/test/runtime/bridge_runtime_cleanup_test.dart +++ b/test/runtime/bridge_runtime_cleanup_test.dart @@ -342,6 +342,7 @@ class _CapturingGatewayRuntimeSessionClient required String method, Map? params, Duration timeout = const Duration(seconds: 15), + bool allowErrorPayload = false, }) { throw UnimplementedError('request is not used by this cleanup test'); } diff --git a/test/runtime/gateway_runtime_bridge_skills_test.dart b/test/runtime/gateway_runtime_bridge_skills_test.dart index bd80d942..84bfd2d4 100644 --- a/test/runtime/gateway_runtime_bridge_skills_test.dart +++ b/test/runtime/gateway_runtime_bridge_skills_test.dart @@ -11,7 +11,7 @@ import 'package:xworkmate/runtime/secure_config_store.dart'; void main() { test( - 'SkillsController lazily connects and loads OpenClaw skills through bridge gateway request', + 'SkillsController loads OpenClaw skills through bridge request without legacy gateway connect', () async { final observedMethods = []; final observedGatewayRequests = >[]; @@ -23,31 +23,6 @@ void main() { observedMethods.add(method); request.response.headers.contentType = ContentType.json; - if (method == 'xworkmate.gateway.connect') { - request.response.write( - jsonEncode({ - 'jsonrpc': '2.0', - 'id': rpc['id'], - 'result': { - 'ok': true, - 'snapshot': { - 'status': 'connected', - 'mode': 'remote', - 'statusText': 'Connected', - 'mainSessionKey': 'main', - }, - 'auth': { - 'role': 'operator', - 'scopes': ['operator.read', 'operator.write'], - }, - 'returnedDeviceToken': '', - }, - }), - ); - await request.response.close(); - return; - } - if (method == 'xworkmate.gateway.request') { final params = (rpc['params'] as Map).cast(); observedGatewayRequests.add(params); @@ -159,10 +134,7 @@ void main() { final controller = SkillsController(runtime); await controller.refresh(agentId: 'main'); - expect(observedMethods, [ - 'xworkmate.gateway.connect', - 'xworkmate.gateway.request', - ]); + expect(observedMethods, const ['xworkmate.gateway.request']); expect(observedGatewayRequests.single['method'], 'skills.status'); expect( (observedGatewayRequests.single['params'] as Map)['agentId'], @@ -176,4 +148,92 @@ void main() { expect(controller.items.first.eligible, isTrue); }, ); + + test( + 'SkillsController keeps bridge skill payload when OpenClaw gateway is offline', + () async { + final observedGatewayRequests = >[]; + final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0); + final subscription = server.listen((request) async { + final body = await utf8.decoder.bind(request).join(); + final rpc = jsonDecode(body) as Map; + request.response.headers.contentType = ContentType.json; + + if (rpc['method'] == 'xworkmate.gateway.request') { + final params = (rpc['params'] as Map).cast(); + observedGatewayRequests.add(params); + request.response.write( + jsonEncode({ + 'jsonrpc': '2.0', + 'id': rpc['id'], + 'result': { + 'ok': false, + 'error': { + 'code': 'OFFLINE', + 'message': 'gateway not connected', + }, + 'payload': { + 'skills': >[ + { + 'name': 'PDF Writer', + 'description': 'Write PDF documents', + 'source': 'openclaw-workspace', + 'skillKey': 'pdf', + 'eligible': false, + 'disabled': false, + 'missing': { + 'bins': [], + 'env': [], + 'config': [], + }, + }, + ], + }, + }, + }), + ); + await request.response.close(); + return; + } + + request.response.statusCode = HttpStatus.badRequest; + await request.response.close(); + }); + + final tempDir = await Directory.systemTemp.createTemp( + 'xworkmate-bridge-skills-offline-test-', + ); + final store = SecureConfigStore( + enableSecureStorage: false, + appDataRootPathResolver: () async => '${tempDir.path}/settings.sqlite3', + secretRootPathResolver: () async => tempDir.path, + ); + final acpClient = GatewayAcpClient( + endpointResolver: () => Uri.parse('http://127.0.0.1:${server.port}'), + authorizationResolver: (_) async => 'bridge-token', + ); + final runtime = GatewayRuntime( + store: store, + identityStore: DeviceIdentityStore(store), + sessionClient: GatewayAcpRuntimeSessionClient(client: acpClient), + ); + await runtime.initialize(); + addTearDown(() async { + runtime.dispose(); + await subscription.cancel(); + await server.close(force: true); + await tempDir.delete(recursive: true); + }); + + final controller = SkillsController(runtime); + await controller.refresh(agentId: 'main'); + + expect(observedGatewayRequests.single['method'], 'skills.status'); + expect(controller.error, isNull); + expect(controller.items.map((item) => item.skillKey), const [ + 'pdf', + ]); + expect(controller.items.single.eligible, isFalse); + }, + ); }