ai-workspace-lab/xworkmate-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: WebRTC remote desktop connection, cleanup local fallback, and ignore .gradle cache

Browse Source
This commit is contained in:
Haitao Pan 2026-06-04 09:12:08 +08:00
parent 5f43ffa188
commit 3d1d037e1a
21 changed files with 711 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -64,3 +64,6 @@ app.*.map.json
/rust/Cargo.lock
/macos/Frameworks/*.dylib
/macos/Frameworks/*.a
# Gradle artifacts (including third_party)
**/.gradle/

5
lib/app/app_controller_desktop_gateway.dart
View File

@ -232,11 +232,6 @@ extension AppControllerDesktopGateway on AppController {
await refreshGatewayHealth();
await refreshAgents();
await refreshSessions();
await skillsControllerInternal.refresh(
agentId: agentsControllerInternal.selectedAgentId.isEmpty
? null
: agentsControllerInternal.selectedAgentId,
);
await modelsControllerInternal.refresh();
await cronJobsControllerInternal.refresh();
await devicesControllerInternal.refresh(quiet: true);

5
lib/app/app_controller_desktop_thread_actions.dart
View File

@ -179,11 +179,6 @@ extension AppControllerDesktopThreadActions on AppController {
if (isAppOwnedAssistantSessionKeyInternal(sessionKey)) {
await chatControllerInternal.loadSession(sessionKey);
}
await skillsControllerInternal.refresh(
agentId: agentsControllerInternal.selectedAgentId.isEmpty
? null
: agentsControllerInternal.selectedAgentId,
);
recomputeTasksInternal();
}

20
lib/features/assistant/assistant_page_composer_skill_picker.dart
View File

@ -75,6 +75,7 @@ class SkillPickerPopoverInternal extends StatelessWidget {
required this.selectedSkillKeys,
required this.filteredSkills,
required this.isLoading,
required this.errorText,
required this.hasQuery,
required this.onQueryChanged,
required this.onToggleSkill,
@ -86,6 +87,7 @@ class SkillPickerPopoverInternal extends StatelessWidget {
final List<String> selectedSkillKeys;
final List<ComposerSkillOptionInternal> filteredSkills;
final bool isLoading;
final String? errorText;
final bool hasQuery;
final ValueChanged<String> onQueryChanged;
final ValueChanged<String> onToggleSkill;
@ -95,6 +97,7 @@ class SkillPickerPopoverInternal extends StatelessWidget {
final palette = context.palette;
final theme = Theme.of(context);
final groupedSkills = skillPickerSectionsInternal(filteredSkills);
final hasError = !isLoading && (errorText?.trim().isNotEmpty ?? false);
return Material(
key: const Key('assistant-skill-picker-popover'),
color: Colors.transparent,
@ -160,6 +163,11 @@ class SkillPickerPopoverInternal extends StatelessWidget {
Text(
isLoading
? appText('正在加载技能…', 'Loading skills…')
: hasError
? appText(
'技能列表加载失败，请稍后重试。',
'Could not load skills. Please try again.',
)
: hasQuery
? appText('没有匹配的技能。', 'No matching skills.')
: appText(
@ -171,6 +179,18 @@ class SkillPickerPopoverInternal extends StatelessWidget {
color: palette.textSecondary,
),
),
if (hasError) ...[
const SizedBox(height: 8),
Text(
errorText!.trim(),
textAlign: TextAlign.center,
maxLines: 3,
overflow: TextOverflow.ellipsis,
style: theme.textTheme.bodySmall?.copyWith(
color: palette.textMuted,
),
),
],
],
),
),

1
lib/features/assistant/assistant_page_composer_state_helpers.dart
View File

@ -95,6 +95,7 @@ Widget buildSkillPickerOverlayForInternal(
selectedSkillKeys: state.widget.selectedSkillKeys,
filteredSkills: state.filteredSkillOptionsInternal(),
isLoading: state.widget.controller.skillsController.loading,
errorText: state.widget.controller.skillsController.error,
hasQuery: state.skillPickerQueryInternal.trim().isNotEmpty,
onQueryChanged: state.setSkillPickerQueryInternal,
onToggleSkill: (skillKey) => state.widget.onToggleSkill(skillKey),

71
lib/features/desktop/desktop_client.dart
View File

@ -3,6 +3,33 @@ import 'dart:convert';
import 'package:flutter_webrtc/flutter_webrtc.dart';
import '../../app/app_controller.dart';
String desktopConnectionStateName(RTCPeerConnectionState state) {
final value = state.toString().split('.').last;
return value.replaceFirst('RTCPeerConnectionState', '').toLowerCase();
}
Map<String, Object?> desktopOfferParams({
required String sessionId,
required String? sdpOffer,
required String display,
required int width,
required int height,
required int fps,
required int bitrate,
required bool useGpu,
}) {
return {
'sessionId': sessionId,
'sdpOffer': sdpOffer,
'display': display,
'width': width,
'height': height,
'fps': fps,
'bitrate': bitrate,
'useGpu': useGpu,
};
}
class DesktopClient {
DesktopClient({required this.controller, required this.sessionId});
@ -42,7 +69,7 @@ class DesktopClient {
try {
final config = {
'iceServers': [
{'url': 'stun:stun.l.google.com:19302'}
{'urls': 'stun:stun.l.google.com:19302'},
],
'sdpSemantics': 'unified-plan',
};
@ -60,18 +87,27 @@ class DesktopClient {
};
_peerConnection!.onConnectionState = (state) {
_stateController.add(state.toString().split('.').last);
_stateController.add(desktopConnectionStateName(state));
};
// Create data channel for inputs BEFORE creating offer
final dcConfig = RTCDataChannelInit()..ordered = true;
_dataChannel =
await _peerConnection!.createDataChannel('input', dcConfig);
_dataChannel = await _peerConnection!.createDataChannel(
'input',
dcConfig,
);
// Handle ICE Candidates generated locally
final List<RTCIceCandidate> iceQueue = [];
bool isRemoteSet = false;
_peerConnection!.onIceCandidate = (RTCIceCandidate? candidate) {
if (candidate != null) {
_sendIceCandidate(candidate);
if (isRemoteSet) {
unawaited(_sendIceCandidate(candidate));
} else {
iceQueue.add(candidate);
}
}
};
@ -88,16 +124,16 @@ class DesktopClient {
// Send SDP Offer to Bridge
final response = await controller.gatewayAcpClientInternal.request(
method: 'xworkmate.desktop.offer',
params: {
'sessionId': sessionId,
'sdpOffer': offer.sdp,
'display': display,
'width': width.toString(),
'height': height.toString(),
'fps': fps.toString(),
'bitrate': bitrate.toString(),
'useGpu': useGpu.toString(),
},
params: desktopOfferParams(
sessionId: sessionId,
sdpOffer: offer.sdp,
display: display,
width: width,
height: height,
fps: fps,
bitrate: bitrate,
useGpu: useGpu,
),
);
final sdpAnswerData = response['result']?['sdpAnswer'];
@ -116,6 +152,11 @@ class DesktopClient {
answer = RTCSessionDescription(sdpAnswerData.toString(), 'answer');
}
await _peerConnection!.setRemoteDescription(answer);
isRemoteSet = true;
for (final candidate in iceQueue) {
unawaited(_sendIceCandidate(candidate));
}
iceQueue.clear();
_isConnecting = false;
} catch (e) {

158
lib/features/desktop/desktop_input_handler.dart
View File

@ -8,43 +8,39 @@ class DesktopInputHandler {
final void Function(Map<String, dynamic> event) onSendInput;
int _lastPressedButton = 1; // Default to left click
void handlePointerMove(PointerEvent event, Size widgetSize) {
if (widgetSize.width == 0 || widgetSize.height == 0) return;
void handlePointerMove(
PointerEvent event,
Size widgetSize, {
Size? contentSize,
}) {
final position = desktopContentPosition(
event.localPosition,
widgetSize,
contentSize: contentSize,
);
if (position == null) return;
final x = event.localPosition.dx / widgetSize.width;
final y = event.localPosition.dy / widgetSize.height;
final cx = x.clamp(0.0, 1.0);
final cy = y.clamp(0.0, 1.0);
onSendInput({
'type': 'mouse_move',
'x': cx,
'y': cy,
});
onSendInput({'type': 'mouse_move', 'x': position.dx, 'y': position.dy});
}
void handlePointerDown(PointerDownEvent event, Size widgetSize) {
if (widgetSize.width == 0 || widgetSize.height == 0) return;
final x = event.localPosition.dx / widgetSize.width;
final y = event.localPosition.dy / widgetSize.height;
final cx = x.clamp(0.0, 1.0);
final cy = y.clamp(0.0, 1.0);
void handlePointerDown(
PointerDownEvent event,
Size widgetSize, {
Size? contentSize,
}) {
final position = desktopContentPosition(
event.localPosition,
widgetSize,
contentSize: contentSize,
);
if (position == null) return;
// Send move event first to ensure click hits the exact coordinates
onSendInput({
'type': 'mouse_move',
'x': cx,
'y': cy,
});
onSendInput({'type': 'mouse_move', 'x': position.dx, 'y': position.dy});
_lastPressedButton = _mapPointerButtons(event.buttons);
onSendInput({
'type': 'mouse_down',
'button': _lastPressedButton,
});
onSendInput({'type': 'mouse_down', 'button': _lastPressedButton});
}
void handlePointerUp(PointerUpEvent event, Size widgetSize) {
@ -55,29 +51,21 @@ class DesktopInputHandler {
releasedButton = _mapPointerButtons(event.buttons);
}
onSendInput({
'type': 'mouse_up',
'button': releasedButton,
});
onSendInput({'type': 'mouse_up', 'button': releasedButton});
}
void handleScroll(PointerScrollEvent event) {
// 4 = scroll up, 5 = scroll down in X11/xdotool button maps
final button = event.scrollDelta.dy < 0 ? 4 : 5;
onSendInput({
'type': 'scroll',
'button': button,
});
onSendInput({'type': 'scroll', 'button': button});
}
void handleKeyEvent(KeyEvent event) {
final isDown = event is KeyDownEvent || event is KeyRepeatEvent;
final keyLabel = event.logicalKey.keyLabel;
final keyLabel = desktopKeyName(event.logicalKey);
if (keyLabel == null) return;
onSendInput({
'type': isDown ? 'key_down' : 'key_up',
'key': keyLabel,
});
onSendInput({'type': isDown ? 'key_down' : 'key_up', 'key': keyLabel});
}
int _mapPointerButtons(int buttons) {
@ -92,3 +80,89 @@ class DesktopInputHandler {
return 1;
}
}
String? desktopKeyName(LogicalKeyboardKey key) {
if (key == LogicalKeyboardKey.enter) return 'Return';
if (key == LogicalKeyboardKey.numpadEnter) return 'Return';
if (key == LogicalKeyboardKey.space) return 'space';
if (key == LogicalKeyboardKey.backspace) return 'BackSpace';
if (key == LogicalKeyboardKey.tab) return 'Tab';
if (key == LogicalKeyboardKey.escape) return 'Escape';
if (key == LogicalKeyboardKey.delete) return 'Delete';
if (key == LogicalKeyboardKey.arrowLeft) return 'Left';
if (key == LogicalKeyboardKey.arrowRight) return 'Right';
if (key == LogicalKeyboardKey.arrowUp) return 'Up';
if (key == LogicalKeyboardKey.arrowDown) return 'Down';
if (key == LogicalKeyboardKey.home) return 'Home';
if (key == LogicalKeyboardKey.end) return 'End';
if (key == LogicalKeyboardKey.pageUp) return 'Page_Up';
if (key == LogicalKeyboardKey.pageDown) return 'Page_Down';
final label = key.keyLabel;
if (label.isEmpty) return null;
if (label.length == 1) {
final punctuation = _xdotoolPunctuationNames[label];
if (punctuation != null) return punctuation;
if (RegExp(r'^[A-Z]$').hasMatch(label)) {
return label.toLowerCase();
}
}
return label;
}
const Map<String, String> _xdotoolPunctuationNames = <String, String>{
'/': 'slash',
'.': 'period',
',': 'comma',
'-': 'minus',
'_': 'underscore',
'=': 'equal',
';': 'semicolon',
"'": 'apostrophe',
'`': 'grave',
'[': 'bracketleft',
']': 'bracketright',
'\\': 'backslash',
};
Offset? desktopContentPosition(
Offset localPosition,
Size viewportSize, {
Size? contentSize,
}) {
if (viewportSize.width <= 0 || viewportSize.height <= 0) return null;
final resolvedContentSize = contentSize;
if (resolvedContentSize == null ||
resolvedContentSize.width <= 0 ||
resolvedContentSize.height <= 0) {
return Offset(
(localPosition.dx / viewportSize.width).clamp(0.0, 1.0),
(localPosition.dy / viewportSize.height).clamp(0.0, 1.0),
);
}
final viewportAspect = viewportSize.width / viewportSize.height;
final contentAspect = resolvedContentSize.width / resolvedContentSize.height;
double drawnWidth;
double drawnHeight;
double offsetX;
double offsetY;
if (viewportAspect > contentAspect) {
drawnHeight = viewportSize.height;
drawnWidth = drawnHeight * contentAspect;
offsetX = (viewportSize.width - drawnWidth) / 2;
offsetY = 0;
} else {
drawnWidth = viewportSize.width;
drawnHeight = drawnWidth / contentAspect;
offsetX = 0;
offsetY = (viewportSize.height - drawnHeight) / 2;
}
return Offset(
((localPosition.dx - offsetX) / drawnWidth).clamp(0.0, 1.0),
((localPosition.dy - offsetY) / drawnHeight).clamp(0.0, 1.0),
);
}

149
lib/features/desktop/desktop_view.dart
View File

@ -29,17 +29,28 @@ class _DesktopViewState extends State<DesktopView> {
DesktopInputHandler? _inputHandler;
// Settings controllers
final TextEditingController _displayController = TextEditingController(text: ':0.0');
final TextEditingController _widthController = TextEditingController(text: '1280');
final TextEditingController _heightController = TextEditingController(text: '720');
final TextEditingController _fpsController = TextEditingController(text: '30');
final TextEditingController _bitrateController = TextEditingController(text: '2000');
final TextEditingController _displayController = TextEditingController(
text: ':0.0',
);
final TextEditingController _widthController = TextEditingController(
text: '1280',
);
final TextEditingController _heightController = TextEditingController(
text: '720',
);
final TextEditingController _fpsController = TextEditingController(
text: '30',
);
final TextEditingController _bitrateController = TextEditingController(
text: '2000',
);
bool _useGpu = false;
bool _showAdvancedOptions = false;
String _connectionState = 'disconnected';
bool _hasStream = false;
bool _isFocused = false;
Size _remoteDesktopSize = const Size(1280, 720);
final FocusNode _viewportFocusNode = FocusNode();
final GlobalKey _viewportKey = GlobalKey();
@ -55,11 +66,13 @@ class _DesktopViewState extends State<DesktopView> {
controller: widget.controller,
sessionId: 'remote-desktop-session',
);
_inputHandler = DesktopInputHandler(onSendInput: (event) {
if (_connectionState == 'connected') {
_client.sendInput(event);
}
});
_inputHandler = DesktopInputHandler(
onSendInput: (event) {
if (_connectionState == 'connected') {
_client.sendInput(event);
}
},
);
_streamSubscription = _client.onRemoteStream.listen((stream) {
if (mounted) {
@ -74,7 +87,8 @@ class _DesktopViewState extends State<DesktopView> {
if (mounted) {
setState(() {
_connectionState = state.toLowerCase();
if (_connectionState == 'disconnected' || _connectionState == 'failed') {
if (_connectionState == 'disconnected' ||
_connectionState == 'failed') {
_hasStream = false;
_localRenderer.srcObject = null;
}
@ -111,6 +125,7 @@ class _DesktopViewState extends State<DesktopView> {
final height = int.tryParse(_heightController.text) ?? 720;
final fps = int.tryParse(_fpsController.text) ?? 30;
final bitrate = int.tryParse(_bitrateController.text) ?? 2000;
_remoteDesktopSize = Size(width.toDouble(), height.toDouble());
try {
await _client.connect(
@ -135,7 +150,8 @@ class _DesktopViewState extends State<DesktopView> {
}
Size _getViewportSize() {
final renderBox = _viewportKey.currentContext?.findRenderObject() as RenderBox?;
final renderBox =
_viewportKey.currentContext?.findRenderObject() as RenderBox?;
return renderBox?.size ?? Size.zero;
}
@ -168,11 +184,16 @@ class _DesktopViewState extends State<DesktopView> {
backgroundColor: _connectionState == 'connected'
? Colors.redAccent
: (_connectionState == 'connecting'
? Colors.orangeAccent
: theme.colorScheme.primary),
? Colors.orangeAccent
: theme.colorScheme.primary),
foregroundColor: Colors.white,
padding: const EdgeInsets.symmetric(horizontal: 20, vertical: 16),
shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(8)),
padding: const EdgeInsets.symmetric(
horizontal: 20,
vertical: 16,
),
shape: RoundedRectangleBorder(
borderRadius: BorderRadius.circular(8),
),
),
icon: Icon(
_connectionState == 'connected'
@ -182,26 +203,31 @@ class _DesktopViewState extends State<DesktopView> {
label: Text(
_connectionState == 'connected'
? '断开连接'
: (_connectionState == 'connecting' ? '正在连接...' : '连接桌面'),
: (_connectionState == 'connecting'
? '正在连接...'
: '连接桌面'),
style: const TextStyle(fontWeight: FontWeight.bold),
),
),
// Status Indicator
Container(
padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 6),
padding: const EdgeInsets.symmetric(
horizontal: 12,
vertical: 6,
),
decoration: BoxDecoration(
color: _connectionState == 'connected'
? Colors.green.withValues(alpha: 0.15)
: (_connectionState == 'connecting'
? Colors.orange.withValues(alpha: 0.15)
: Colors.grey.withValues(alpha: 0.15)),
? Colors.orange.withValues(alpha: 0.15)
: Colors.grey.withValues(alpha: 0.15)),
borderRadius: BorderRadius.circular(16),
border: Border.all(
color: _connectionState == 'connected'
? Colors.green
: (_connectionState == 'connecting'
? Colors.orange
: Colors.grey),
? Colors.orange
: Colors.grey),
width: 1,
),
),
@ -215,8 +241,8 @@ class _DesktopViewState extends State<DesktopView> {
color: _connectionState == 'connected'
? Colors.green
: (_connectionState == 'connecting'
? Colors.orange
: Colors.grey),
? Colors.orange
: Colors.grey),
shape: BoxShape.circle,
),
),
@ -224,15 +250,19 @@ class _DesktopViewState extends State<DesktopView> {
Text(
_connectionState == 'connected'
? '已连接'
: (_connectionState == 'connecting' ? '连接中' : '未连接'),
: (_connectionState == 'connecting'
? '连接中'
: '未连接'),
style: TextStyle(
fontSize: 12,
fontWeight: FontWeight.bold,
color: _connectionState == 'connected'
? Colors.green
: (_connectionState == 'connecting'
? Colors.orange
: (isDark ? Colors.white70 : Colors.black87)),
? Colors.orange
: (isDark
? Colors.white70
: Colors.black87)),
),
),
],
@ -240,15 +270,25 @@ class _DesktopViewState extends State<DesktopView> {
),
// Advanced Options Toggle
TextButton.icon(
onPressed: () => setState(() => _showAdvancedOptions = !_showAdvancedOptions),
icon: Icon(_showAdvancedOptions ? Icons.expand_less : Icons.expand_more),
onPressed: () => setState(
() => _showAdvancedOptions = !_showAdvancedOptions,
),
icon: Icon(
_showAdvancedOptions
? Icons.expand_less
: Icons.expand_more,
),
label: const Text('高级选项'),
),
// Maximize Toggle
if (widget.onToggleMaximize != null)
IconButton(
onPressed: widget.onToggleMaximize,
icon: Icon(widget.isMaximized ? Icons.fullscreen_exit_rounded : Icons.fullscreen_rounded),
icon: Icon(
widget.isMaximized
? Icons.fullscreen_exit_rounded
: Icons.fullscreen_rounded,
),
tooltip: widget.isMaximized ? '恢复默认大小' : '最大化',
),
],
@ -345,7 +385,9 @@ class _DesktopViewState extends State<DesktopView> {
});
},
onKeyEvent: (node, event) {
if (_isFocused && _connectionState == 'connected' && _inputHandler != null) {
if (_isFocused &&
_connectionState == 'connected' &&
_inputHandler != null) {
_inputHandler!.handleKeyEvent(event);
return KeyEventResult.handled;
}
@ -354,7 +396,9 @@ class _DesktopViewState extends State<DesktopView> {
child: Container(
key: _viewportKey,
decoration: BoxDecoration(
color: isDark ? Colors.black26 : Colors.black.withValues(alpha: 0.04),
color: isDark
? Colors.black26
: Colors.black.withValues(alpha: 0.04),
borderRadius: BorderRadius.circular(12),
border: Border.all(
color: _isFocused
@ -373,12 +417,20 @@ class _DesktopViewState extends State<DesktopView> {
behavior: HitTestBehavior.opaque,
onPointerHover: (event) {
if (_inputHandler != null) {
_inputHandler!.handlePointerMove(event, _getViewportSize());
_inputHandler!.handlePointerMove(
event,
_getViewportSize(),
contentSize: _remoteDesktopSize,
);
}
},
onPointerMove: (event) {
if (_inputHandler != null) {
_inputHandler!.handlePointerMove(event, _getViewportSize());
_inputHandler!.handlePointerMove(
event,
_getViewportSize(),
contentSize: _remoteDesktopSize,
);
}
},
onPointerDown: (event) {
@ -386,22 +438,31 @@ class _DesktopViewState extends State<DesktopView> {
_viewportFocusNode.requestFocus();
}
if (_inputHandler != null) {
_inputHandler!.handlePointerDown(event, _getViewportSize());
_inputHandler!.handlePointerDown(
event,
_getViewportSize(),
contentSize: _remoteDesktopSize,
);
}
},
onPointerUp: (event) {
if (_inputHandler != null) {
_inputHandler!.handlePointerUp(event, _getViewportSize());
_inputHandler!.handlePointerUp(
event,
_getViewportSize(),
);
}
},
onPointerSignal: (event) {
if (event is PointerScrollEvent && _inputHandler != null) {
if (event is PointerScrollEvent &&
_inputHandler != null) {
_inputHandler!.handleScroll(event);
}
},
child: RTCVideoView(
_localRenderer,
objectFit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain,
objectFit: RTCVideoViewObjectFit
.RTCVideoViewObjectFitContain,
),
),
),
@ -418,7 +479,9 @@ class _DesktopViewState extends State<DesktopView> {
Icon(
Icons.monitor_rounded,
size: 64,
color: theme.colorScheme.onSurface.withValues(alpha: 0.2),
color: theme.colorScheme.onSurface.withValues(
alpha: 0.2,
),
),
const SizedBox(height: 16),
Text(
@ -426,7 +489,8 @@ class _DesktopViewState extends State<DesktopView> {
? '正在建立 WebRTC 连接，请稍候...'
: '未开启远程桌面流。点击“连接桌面”启动视频流。',
style: TextStyle(
color: theme.colorScheme.onSurface.withValues(alpha: 0.6),
color: theme.colorScheme.onSurface
.withValues(alpha: 0.6),
fontSize: 14,
),
),
@ -449,7 +513,10 @@ class _DesktopViewState extends State<DesktopView> {
opacity: _isFocused ? 0.3 : 0.8,
duration: const Duration(milliseconds: 200),
child: Container(
padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4),
padding: const EdgeInsets.symmetric(
horizontal: 8,
vertical: 4,
),
decoration: BoxDecoration(
color: Colors.black.withValues(alpha: 0.6),
borderRadius: BorderRadius.circular(4),

1
lib/features/settings/settings_logs_panel.dart
View File

@ -56,6 +56,7 @@ class _SettingsLogsPanelState extends State<SettingsLogsPanel> {
}
}
} catch (e) {
appLog('Failed to fetch system status: $e');
if (mounted) {
setState(() {
_bridgeStatus = 'error';

6
lib/runtime/gateway_acp_client.dart
View File

@ -1411,6 +1411,7 @@ class GatewayAcpRuntimeSessionClient implements GatewayRuntimeSessionClient {
required String method,
Map<String, dynamic>? params,
Duration timeout = const Duration(seconds: 15),
bool allowErrorPayload = false,
}) async {
final envelope = await client.request(
method: 'xworkmate.gateway.request',
@ -1423,6 +1424,11 @@ class GatewayAcpRuntimeSessionClient implements GatewayRuntimeSessionClient {
onNotification: _handleNotification,
);
final result = client.asMap(envelope['result']);
if (allowErrorPayload &&
!(client.boolValue(result['ok']) ?? false) &&
result.containsKey('payload')) {
return result['payload'];
}
_throwGatewayResultIfNeeded(result, '$method request failed');
return result['payload'];
}

19
lib/runtime/gateway_runtime_api.dart
View File

@ -199,14 +199,19 @@ extension GatewayRuntimeApiInternal on GatewayRuntime {
Future<List<GatewaySkillSummary>> _listSkillsInternal({
String? agentId,
}) async {
final params = <String, dynamic>{
if (agentId != null && agentId.trim().isNotEmpty)
'agentId': agentId.trim(),
};
final payload = asMap(
await request(
'skills.status',
params: <String, dynamic>{
if (agentId != null && agentId.trim().isNotEmpty)
'agentId': agentId.trim(),
},
),
sessionClientInternal == null
? await request('skills.status', params: params)
: await sessionClientInternal!.request(
runtimeId: runtimeIdInternal,
method: 'skills.status',
params: params,
allowErrorPayload: true,
),
);
return asList(payload['skills'])
.map((item) {

1
lib/runtime/gateway_runtime_session_client.dart
View File

@ -186,6 +186,7 @@ abstract class GatewayRuntimeSessionClient {
required String method,
Map<String, dynamic>? params,
Duration timeout = const Duration(seconds: 15),
bool allowErrorPayload = false,
});
Future<void> disconnect({required String runtimeId});

3
lib/runtime/runtime_controllers_entities.dart
View File

@ -37,9 +37,6 @@ class SkillsController extends ChangeNotifier {
errorInternal = null;
notifyListeners();
try {
await runtimeInternal.ensureBridgeSessionConnected(
selectedAgentId: agentId?.trim() ?? '',
);
itemsInternal = await runtimeInternal.listSkills(agentId: agentId);
} catch (error) {
errorInternal = error.toString();

2
macos/Runner/DebugProfile.entitlements
View File

@ -2,6 +2,8 @@
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.network.client</key>
<true/>
<key>com.apple.security.files.bookmarks.app-scope</key>
<true/>
<key>com.apple.security.files.user-selected.read-only</key>

2
macos/Runner/Release.entitlements
View File

@ -2,6 +2,8 @@
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.network.client</key>
<true/>
<key>com.apple.security.files.bookmarks.app-scope</key>
<true/>
</dict>

34
test/features/assistant/assistant_lower_pane_test.dart
View File

@ -635,6 +635,7 @@ void main() {
),
],
isLoading: false,
errorText: null,
hasQuery: false,
onQueryChanged: (_) {},
onToggleSkill: toggledKeys.add,
@ -684,6 +685,7 @@ void main() {
),
],
isLoading: false,
errorText: null,
hasQuery: true,
onQueryChanged: (_) {},
onToggleSkill: (_) {},
@ -697,6 +699,38 @@ void main() {
expect(find.text('Workspace Skills'), findsNothing);
expect(find.text('Agent Skills'), findsNothing);
});
testWidgets(
'empty skill picker shows refresh error instead of empty state',
(tester) async {
final searchController = TextEditingController();
final focusNode = FocusNode();
addTearDown(searchController.dispose);
addTearDown(focusNode.dispose);
await tester.pumpWidget(
_buildTestApp(
child: SkillPickerPopoverInternal(
maxHeight: 360,
searchController: searchController,
searchFocusNode: focusNode,
selectedSkillKeys: const <String>[],
filteredSkills: const <ComposerSkillOptionInternal>[],
isLoading: false,
errorText: 'skills.status request failed',
hasQuery: false,
onQueryChanged: (_) {},
onToggleSkill: (_) {},
),
),
);
await tester.pumpAndSettle();
expect(find.text('技能列表加载失败，请稍后重试。'), findsOneWidget);
expect(find.text('skills.status request failed'), findsOneWidget);
expect(find.text('当前没有已加载技能。'), findsNothing);
},
);
});
}

46
test/features/desktop/desktop_client_test.dart Normal file
View File

@ -0,0 +1,46 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart';
import 'package:xworkmate/features/desktop/desktop_client.dart';
void main() {
group('DesktopClient protocol helpers', () {
test('normalizes WebRTC connection states for view gating', () {
expect(
desktopConnectionStateName(
RTCPeerConnectionState.RTCPeerConnectionStateConnected,
),
'connected',
);
expect(
desktopConnectionStateName(
RTCPeerConnectionState.RTCPeerConnectionStateFailed,
),
'failed',
);
});
test('builds desktop offer params with native numeric values', () {
final params = desktopOfferParams(
sessionId: 'desktop-session-1',
sdpOffer: 'v=0',
display: ':0.0',
width: 1280,
height: 720,
fps: 30,
bitrate: 2000,
useGpu: false,
);
expect(params['sessionId'], 'desktop-session-1');
expect(params['sdpOffer'], 'v=0');
expect(params['display'], ':0.0');
expect(params['width'], isA<int>());
expect(params['height'], isA<int>());
expect(params['fps'], isA<int>());
expect(params['bitrate'], isA<int>());
expect(params['useGpu'], isA<bool>());
expect(params['width'], 1280);
expect(params['height'], 720);
});
});
}

76
test/features/desktop/desktop_input_handler_test.dart Normal file
View File

@ -0,0 +1,76 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:flutter/services.dart';
import 'package:xworkmate/features/desktop/desktop_input_handler.dart';
void main() {
group('desktopKeyName', () {
test('uses xdotool-compatible names for basic terminal keys', () {
expect(desktopKeyName(LogicalKeyboardKey.enter), 'Return');
expect(desktopKeyName(LogicalKeyboardKey.numpadEnter), 'Return');
expect(desktopKeyName(LogicalKeyboardKey.space), 'space');
expect(desktopKeyName(LogicalKeyboardKey.backspace), 'BackSpace');
expect(desktopKeyName(LogicalKeyboardKey.tab), 'Tab');
expect(desktopKeyName(LogicalKeyboardKey.escape), 'Escape');
});
test('uses xdotool-compatible names for navigation keys', () {
expect(desktopKeyName(LogicalKeyboardKey.arrowLeft), 'Left');
expect(desktopKeyName(LogicalKeyboardKey.arrowRight), 'Right');
expect(desktopKeyName(LogicalKeyboardKey.arrowUp), 'Up');
expect(desktopKeyName(LogicalKeyboardKey.arrowDown), 'Down');
expect(desktopKeyName(LogicalKeyboardKey.pageUp), 'Page_Up');
expect(desktopKeyName(LogicalKeyboardKey.pageDown), 'Page_Down');
});
test('keeps printable keys available for shell and browser text input', () {
expect(desktopKeyName(LogicalKeyboardKey.keyA), 'a');
expect(desktopKeyName(LogicalKeyboardKey.digit1), '1');
expect(desktopKeyName(LogicalKeyboardKey.period), 'period');
expect(desktopKeyName(LogicalKeyboardKey.slash), 'slash');
expect(desktopKeyName(LogicalKeyboardKey.minus), 'minus');
});
});
group('desktopContentPosition', () {
test('maps directly when viewport and content share an aspect ratio', () {
final position = desktopContentPosition(
const Offset(640, 360),
const Size(1280, 720),
contentSize: const Size(1280, 720),
);
expect(position, const Offset(0.5, 0.5));
});
test('subtracts horizontal letterbox before normalizing pointer input', () {
final position = desktopContentPosition(
const Offset(160, 360),
const Size(1600, 720),
contentSize: const Size(1280, 720),
);
expect(position!.dx, closeTo(0.0, 0.001));
expect(position.dy, closeTo(0.5, 0.001));
});
test('subtracts vertical letterbox before normalizing pointer input', () {
final position = desktopContentPosition(
const Offset(640, 140),
const Size(1280, 1000),
contentSize: const Size(1280, 720),
);
expect(position!.dx, closeTo(0.5, 0.001));
expect(position.dy, closeTo(0.0, 0.001));
});
test('keeps legacy full-viewport mapping when content size is unknown', () {
final position = desktopContentPosition(
const Offset(640, 360),
const Size(1280, 720),
);
expect(position, const Offset(0.5, 0.5));
});
});
}

132
test/runtime/assistant_execution_target_test.dart
View File

@ -659,6 +659,138 @@ void main() {
},
);
test(
'skill picker selection allows multiple skills and independent deselect',
() async {
final controller = _connectedGatewayController(
_RecordingGoTaskServiceClient(),
);
addTearDown(controller.dispose);
controller.skillsControllerInternal.itemsInternal =
const <GatewaySkillSummary>[
GatewaySkillSummary(
name: 'PDF Writer',
description: 'Write PDF documents',
source: 'openclaw-workspace',
skillKey: 'pdf',
primaryEnv: null,
eligible: true,
disabled: false,
missingBins: <String>[],
missingEnv: <String>[],
missingConfig: <String>[],
),
GatewaySkillSummary(
name: 'Browser Automation',
description: 'Use browser automation',
source: 'agents-skills-personal',
skillKey: 'browser-automation',
primaryEnv: null,
eligible: true,
disabled: false,
missingBins: <String>[],
missingEnv: <String>[],
missingConfig: <String>[],
),
];
await _selectGatewaySession(controller, 'unit-skill-multi-select-task');
await controller.toggleAssistantSkillForSession(
'unit-skill-multi-select-task',
'pdf',
);
await controller.toggleAssistantSkillForSession(
'unit-skill-multi-select-task',
'browser-automation',
);
expect(
controller.assistantSelectedSkillKeysForSession(
'unit-skill-multi-select-task',
),
const <String>['pdf', 'browser-automation'],
);
await controller.toggleAssistantSkillForSession(
'unit-skill-multi-select-task',
'browser-automation',
);
expect(
controller.assistantSelectedSkillKeysForSession(
'unit-skill-multi-select-task',
),
const <String>['pdf'],
);
},
);
test('skill selection is isolated across task sessions', () async {
final controller = _connectedGatewayController(
_RecordingGoTaskServiceClient(),
);
addTearDown(controller.dispose);
controller.skillsControllerInternal.itemsInternal =
const <GatewaySkillSummary>[
GatewaySkillSummary(
name: 'PDF Writer',
description: 'Write PDF documents',
source: 'openclaw-workspace',
skillKey: 'pdf',
primaryEnv: null,
eligible: true,
disabled: false,
missingBins: <String>[],
missingEnv: <String>[],
missingConfig: <String>[],
),
GatewaySkillSummary(
name: 'Browser Automation',
description: 'Use browser automation',
source: 'agents-skills-personal',
skillKey: 'browser-automation',
primaryEnv: null,
eligible: true,
disabled: false,
missingBins: <String>[],
missingEnv: <String>[],
missingConfig: <String>[],
),
];
await _selectGatewaySession(controller, 'unit-skill-isolated-a');
await controller.toggleAssistantSkillForSession(
'unit-skill-isolated-a',
'pdf',
);
await _selectGatewaySession(controller, 'unit-skill-isolated-b');
expect(
controller.assistantSelectedSkillKeysForSession(
'unit-skill-isolated-b',
),
isEmpty,
);
await controller.toggleAssistantSkillForSession(
'unit-skill-isolated-b',
'browser-automation',
);
expect(
controller
.buildExternalAcpRoutingForSessionInternal('unit-skill-isolated-a')
.explicitSkills,
const <String>['pdf'],
);
expect(
controller
.buildExternalAcpRoutingForSessionInternal('unit-skill-isolated-b')
.explicitSkills,
const <String>['browser-automation'],
);
});
test('skill selection ignores stale non-bridge skill keys', () {
final controller = AppController(
environmentOverride: const <String, String>{},

1
test/runtime/bridge_runtime_cleanup_test.dart
View File

@ -342,6 +342,7 @@ class _CapturingGatewayRuntimeSessionClient
required String method,
Map<String, dynamic>? params,
Duration timeout = const Duration(seconds: 15),
bool allowErrorPayload = false,
}) {
throw UnimplementedError('request is not used by this cleanup test');
}

120
test/runtime/gateway_runtime_bridge_skills_test.dart
View File

@ -11,7 +11,7 @@ import 'package:xworkmate/runtime/secure_config_store.dart';
void main() {
test(
'SkillsController lazily connects and loads OpenClaw skills through bridge gateway request',
'SkillsController loads OpenClaw skills through bridge request without legacy gateway connect',
() async {
final observedMethods = <String>[];
final observedGatewayRequests = <Map<String, dynamic>>[];
@ -23,31 +23,6 @@ void main() {
observedMethods.add(method);
request.response.headers.contentType = ContentType.json;
if (method == 'xworkmate.gateway.connect') {
request.response.write(
jsonEncode(<String, dynamic>{
'jsonrpc': '2.0',
'id': rpc['id'],
'result': <String, dynamic>{
'ok': true,
'snapshot': <String, dynamic>{
'status': 'connected',
'mode': 'remote',
'statusText': 'Connected',
'mainSessionKey': 'main',
},
'auth': <String, dynamic>{
'role': 'operator',
'scopes': <String>['operator.read', 'operator.write'],
},
'returnedDeviceToken': '',
},
}),
);
await request.response.close();
return;
}
if (method == 'xworkmate.gateway.request') {
final params = (rpc['params'] as Map).cast<String, dynamic>();
observedGatewayRequests.add(params);
@ -159,10 +134,7 @@ void main() {
final controller = SkillsController(runtime);
await controller.refresh(agentId: 'main');
expect(observedMethods, <String>[
'xworkmate.gateway.connect',
'xworkmate.gateway.request',
]);
expect(observedMethods, const <String>['xworkmate.gateway.request']);
expect(observedGatewayRequests.single['method'], 'skills.status');
expect(
(observedGatewayRequests.single['params'] as Map)['agentId'],
@ -176,4 +148,92 @@ void main() {
expect(controller.items.first.eligible, isTrue);
},
);
test(
'SkillsController keeps bridge skill payload when OpenClaw gateway is offline',
() async {
final observedGatewayRequests = <Map<String, dynamic>>[];
final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
final subscription = server.listen((request) async {
final body = await utf8.decoder.bind(request).join();
final rpc = jsonDecode(body) as Map<String, dynamic>;
request.response.headers.contentType = ContentType.json;
if (rpc['method'] == 'xworkmate.gateway.request') {
final params = (rpc['params'] as Map).cast<String, dynamic>();
observedGatewayRequests.add(params);
request.response.write(
jsonEncode(<String, dynamic>{
'jsonrpc': '2.0',
'id': rpc['id'],
'result': <String, dynamic>{
'ok': false,
'error': <String, dynamic>{
'code': 'OFFLINE',
'message': 'gateway not connected',
},
'payload': <String, dynamic>{
'skills': <Map<String, dynamic>>[
<String, dynamic>{
'name': 'PDF Writer',
'description': 'Write PDF documents',
'source': 'openclaw-workspace',
'skillKey': 'pdf',
'eligible': false,
'disabled': false,
'missing': <String, dynamic>{
'bins': <String>[],
'env': <String>[],
'config': <String>[],
},
},
],
},
},
}),
);
await request.response.close();
return;
}
request.response.statusCode = HttpStatus.badRequest;
await request.response.close();
});
final tempDir = await Directory.systemTemp.createTemp(
'xworkmate-bridge-skills-offline-test-',
);
final store = SecureConfigStore(
enableSecureStorage: false,
appDataRootPathResolver: () async => '${tempDir.path}/settings.sqlite3',
secretRootPathResolver: () async => tempDir.path,
);
final acpClient = GatewayAcpClient(
endpointResolver: () => Uri.parse('http://127.0.0.1:${server.port}'),
authorizationResolver: (_) async => 'bridge-token',
);
final runtime = GatewayRuntime(
store: store,
identityStore: DeviceIdentityStore(store),
sessionClient: GatewayAcpRuntimeSessionClient(client: acpClient),
);
await runtime.initialize();
addTearDown(() async {
runtime.dispose();
await subscription.cancel();
await server.close(force: true);
await tempDir.delete(recursive: true);
});
final controller = SkillsController(runtime);
await controller.refresh(agentId: 'main');
expect(observedGatewayRequests.single['method'], 'skills.status');
expect(controller.error, isNull);
expect(controller.items.map((item) => item.skillKey), const <String>[
'pdf',
]);
expect(controller.items.single.eligible, isFalse);
},
);
}