From c9e3e67f208516337e6d6645c7b72f5a757a36a8 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sat, 7 Jun 2025 00:05:21 -0600 Subject: [PATCH] Implemented Live Macro Window Detection --- Data/Agent/borealis-agent.py | 140 ++++++++--- .../WebUI/src/Node_Configuration_Sidebar.jsx | 107 +++++--- .../Agent Roles/Node_Agent_Role_Macro.jsx | 232 ++++++++++++++++-- Data/Server/server.py | 34 +++ 4 files changed, 427 insertions(+), 86 deletions(-) diff --git a/Data/Agent/borealis-agent.py b/Data/Agent/borealis-agent.py index 2b80649..a1b7079 100644 --- a/Data/Agent/borealis-agent.py +++ b/Data/Agent/borealis-agent.py @@ -10,6 +10,7 @@ from functools import partial from io import BytesIO import base64 import traceback +import random # Macro Randomization import platform # OS Detection import importlib.util @@ -338,7 +339,6 @@ class ScreenshotRegion(QtWidgets.QWidget): async def screenshot_task(cfg): nid=cfg.get('node_id') alias=cfg.get('alias','') - print(f"[DEBUG] Running screenshot_task for {nid}") r=CONFIG.data['regions'].get(nid) if r: region=(r['x'],r['y'],r['w'],r['h']) @@ -369,40 +369,116 @@ async def screenshot_task(cfg): # ---------------- Macro Task ---------------- async def macro_task(cfg): + """ + Improved macro_task supporting all operation modes, live config, error reporting, and UI feedback. + """ nid = cfg.get('node_id') - window_handle = cfg.get('window_handle') - mode = cfg.get('operation_mode', 'keypress') # 'keypress' or 'typed_text' - key = cfg.get('key') - text = cfg.get('text') - interval_ms = int(cfg.get('interval_ms', 1000)) - randomize = cfg.get('randomize_interval', False) - random_min = int(cfg.get('random_min', 750)) - random_max = int(cfg.get('random_max', 950)) - active = cfg.get('active', True) # Whether macro is "started" or "paused" - print(f"[DEBUG] Macro task for node {nid} started on window {window_handle}") - import random - try: - while True: - if not active: - await asyncio.sleep(0.2) - continue - if mode == 'keypress' and key: - macro_engines.send_keypress_to_window(window_handle, key) - elif mode == 'typed_text' and text: - macro_engines.type_text_to_window(window_handle, text) - # Interval logic - if randomize: - ms = random.randint(random_min, random_max) + # Track trigger state for edge/level changes + last_trigger_value = 0 + has_run_once = False + + while True: + # Always re-fetch config (hot reload support) + # (In reality, you might want to deep-copy or re-provision on config update, but for MVP we refetch each tick) + window_handle = cfg.get('window_handle') + macro_type = cfg.get('macro_type', 'keypress') # Now matches UI config + operation_mode = cfg.get('operation_mode', 'Continuous') + key = cfg.get('key') + text = cfg.get('text') + interval_ms = int(cfg.get('interval_ms', 1000)) + randomize = cfg.get('randomize_interval', False) + random_min = int(cfg.get('random_min', 750)) + random_max = int(cfg.get('random_max', 950)) + active = cfg.get('active', True) + trigger = int(cfg.get('trigger', 0)) # For trigger modes; default 0 if not set + + # Define helper for error reporting + async def emit_macro_status(success, message=""): + await sio.emit('macro_status', { + "agent_id": AGENT_ID, + "node_id": nid, + "success": success, + "message": message, + "timestamp": int(asyncio.get_event_loop().time() * 1000) + }) + + # Stopped state (paused from UI) + if not (active is True or str(active).lower() == "true"): + await asyncio.sleep(0.2) + continue + + try: + send_macro = False + + # Operation Mode Logic + if operation_mode == "Run Once": + if not has_run_once: + send_macro = True + has_run_once = True # Only run once, then stop + elif operation_mode == "Continuous": + send_macro = True # Always run every interval + elif operation_mode == "Trigger-Continuous": + # Only run while trigger is "1" + if trigger == 1: + send_macro = True + else: + send_macro = False + elif operation_mode == "Trigger-Once": + # Run only on rising edge: 0->1 + if last_trigger_value == 0 and trigger == 1: + send_macro = True + else: + send_macro = False + last_trigger_value = trigger else: - ms = interval_ms - await asyncio.sleep(ms / 1000.0) - except asyncio.CancelledError: - print(f"[TASK] Macro role {nid} cancelled.") - except Exception as e: - print(f"[ERROR] Macro task {nid} failed: {e}") - import traceback - traceback.print_exc() + # Unknown mode: default to "Continuous" + send_macro = True + + if send_macro: + # Actually perform macro + if macro_type == 'keypress' and key: + result = macro_engines.send_keypress_to_window(window_handle, key) + elif macro_type == 'typed_text' and text: + result = macro_engines.type_text_to_window(window_handle, text) + else: + await emit_macro_status(False, "Invalid macro type or missing key/text") + await asyncio.sleep(0.2) + continue + + # Result may be True or (False, error) + if isinstance(result, tuple): + success, err = result + else: + success, err = bool(result), "" + + if success: + await emit_macro_status(True, f"Macro sent: {macro_type}") + else: + await emit_macro_status(False, err or "Unknown macro engine failure") + else: + # No macro to send this cycle, just idle + await asyncio.sleep(0.05) + + # Timing: only wait if we did send macro this tick + if send_macro: + if randomize: + ms = random.randint(random_min, random_max) + else: + ms = interval_ms + await asyncio.sleep(ms / 1000.0) + else: + await asyncio.sleep(0.1) # No macro action: check again soon + + except asyncio.CancelledError: + print(f"[TASK] Macro role {nid} cancelled.") + break + except Exception as e: + print(f"[ERROR] Macro task {nid} failed: {e}") + import traceback + traceback.print_exc() + await emit_macro_status(False, str(e)) + await asyncio.sleep(0.5) # ---------------- Config Watcher ---------------- async def config_watcher(): diff --git a/Data/Server/WebUI/src/Node_Configuration_Sidebar.jsx b/Data/Server/WebUI/src/Node_Configuration_Sidebar.jsx index 82afefb..bf3298c 100644 --- a/Data/Server/WebUI/src/Node_Configuration_Sidebar.jsx +++ b/Data/Server/WebUI/src/Node_Configuration_Sidebar.jsx @@ -43,13 +43,27 @@ export default function NodeConfigurationSidebar({ drawerOpen, setDrawerOpen, ti return config.map((field, index) => { const value = nodeData?.[field.key] || ""; - return ( - - - {field.label || field.key} - + // ---- DYNAMIC DROPDOWN SUPPORT ---- + if (field.type === "select") { + let options = field.options || []; - {field.type === "select" ? ( + // Handle dynamic options for things like Target Window + if (field.dynamicOptions && nodeData?.windowList && Array.isArray(nodeData.windowList)) { + options = nodeData.windowList + .map(win => ({ + value: String(win.handle), + label: `${win.title} (${win.handle})` + })) + .sort((a, b) => a.label.localeCompare(b.label, undefined, { sensitivity: "base" })); + } else { + options = options.map(opt => ({ value: opt, label: opt })); + } + + return ( + + + {field.label || field.key} + - {(field.options || []).map((opt, idx) => ( - - {opt} + {options.length === 0 ? ( + + {field.label === "Target Window" + ? "No windows detected" + : "No options"} - ))} + ) : ( + options.map((opt, idx) => ( + + {opt.label} + + )) + )} + + ); + } + // ---- END DYNAMIC DROPDOWN SUPPORT ---- - ) : ( - { - const newValue = e.target.value; - if (!nodeId) return; - effectiveSetNodes((nds) => - nds.map((n) => - n.id === nodeId - ? { ...n, data: { ...n.data, [field.key]: newValue } } - : n - ) - ); - window.BorealisValueBus[nodeId] = newValue; - }} - InputProps={{ - sx: { - backgroundColor: "#1e1e1e", - color: "#ccc", - "& fieldset": { borderColor: "#444" }, - "&:hover fieldset": { borderColor: "#666" }, - "&.Mui-focused fieldset": { borderColor: "#58a6ff" } - } - }} - /> - )} + return ( + + + {field.label || field.key} + + { + const newValue = e.target.value; + if (!nodeId) return; + effectiveSetNodes((nds) => + nds.map((n) => + n.id === nodeId + ? { ...n, data: { ...n.data, [field.key]: newValue } } + : n + ) + ); + window.BorealisValueBus[nodeId] = newValue; + }} + InputProps={{ + sx: { + backgroundColor: "#1e1e1e", + color: "#ccc", + "& fieldset": { borderColor: "#444" }, + "&:hover fieldset": { borderColor: "#666" }, + "&.Mui-focused fieldset": { borderColor: "#58a6ff" } + } + }} + /> ); }); diff --git a/Data/Server/WebUI/src/nodes/Agent Roles/Node_Agent_Role_Macro.jsx b/Data/Server/WebUI/src/nodes/Agent Roles/Node_Agent_Role_Macro.jsx index 6901202..6bf42a3 100644 --- a/Data/Server/WebUI/src/nodes/Agent Roles/Node_Agent_Role_Macro.jsx +++ b/Data/Server/WebUI/src/nodes/Agent Roles/Node_Agent_Role_Macro.jsx @@ -17,21 +17,130 @@ const OPERATION_MODES = [ "Trigger-Continuous" ]; +const MACRO_TYPES = [ + "keypress", + "typed_text" +]; + +const statusColors = { + idle: "#333", + running: "#00d18c", + error: "#ff4f4f", + success: "#00d18c" +}; + const MacroKeyPressNode = ({ id, data }) => { const { setNodes, getNodes } = useReactFlow(); const edges = useStore((state) => state.edges); + const [windowList, setWindowList] = useState([]); + const [status, setStatus] = useState({ state: "idle", message: "" }); + const socketRef = useRef(null); // Determine if agent is connected const agentEdge = edges.find((e) => e.target === id && e.targetHandle === "agent"); const agentNode = agentEdge && getNodes().find((n) => n.id === agentEdge.source); const agentConnection = !!(agentNode && agentNode.data && agentNode.data.agent_id); + const agent_id = agentNode && agentNode.data && agentNode.data.agent_id; // Macro run/trigger state (sidebar sets this via config, but node UI just shows status) const running = data?.active === true || data?.active === "true"; - // Node UI (no config fields, only status) + // Store for last macro error/status + const [lastMacroStatus, setLastMacroStatus] = useState({ success: true, message: "", timestamp: null }); + + // Setup WebSocket for agent macro status updates + useEffect(() => { + if (!window.BorealisSocket) return; + const socket = window.BorealisSocket; + socketRef.current = socket; + + function handleMacroStatus(payload) { + if ( + payload && + payload.agent_id === agent_id && + payload.node_id === id + ) { + setLastMacroStatus({ + success: !!payload.success, + message: payload.message || "", + timestamp: payload.timestamp || Date.now() + }); + setStatus({ + state: payload.success ? "success" : "error", + message: payload.message || (payload.success ? "Success" : "Error") + }); + } + } + + socket.on("macro_status", handleMacroStatus); + return () => { + socket.off("macro_status", handleMacroStatus); + }; + }, [agent_id, id]); + + // Auto-refresh window list from agent + useEffect(() => { + let intervalId = null; + async function fetchWindows() { + if (window.BorealisSocket && agentConnection) { + window.BorealisSocket.emit("list_agent_windows", { + agent_id + }); + } + } + fetchWindows(); + intervalId = setInterval(fetchWindows, WINDOW_LIST_REFRESH_MS); + + // Listen for agent_window_list updates + function handleAgentWindowList(payload) { + if (payload?.agent_id === agent_id && Array.isArray(payload.windows)) { + setWindowList(payload.windows); + + // Store windowList in node data for sidebar dynamic dropdowns + setNodes(nds => + nds.map(n => + n.id === id + ? { ...n, data: { ...n.data, windowList: payload.windows } } + : n + ) + ); + } + } + if (window.BorealisSocket) { + window.BorealisSocket.on("agent_window_list", handleAgentWindowList); + } + + return () => { + clearInterval(intervalId); + if (window.BorealisSocket) { + window.BorealisSocket.off("agent_window_list", handleAgentWindowList); + } + }; + }, [agent_id, agentConnection, setNodes, id]); + + // UI: Start/Pause Button + const handleToggleMacro = () => { + setNodes(nds => + nds.map(n => + n.id === id + ? { + ...n, + data: { + ...n.data, + active: n.data?.active === true || n.data?.active === "true" ? "false" : "true" + } + } + : n + ) + ); + }; + + // Optional: Show which window is targeted by name + const selectedWindow = (windowList || []).find(w => String(w.handle) === String(data?.window_handle)); + + // Node UI (no config fields, only status + window list) return ( -
+
{/* --- INPUT LABELS & HANDLES --- */}
{ transform: "translateY(-50%)", height: "10px", borderRadius: "50%", - backgroundColor: running ? "#00d18c" : "#333", + backgroundColor: + status.state === "error" + ? statusColors.error + : running + ? statusColors.running + : statusColors.idle, border: "1px solid #222" }} />
- Status: {running ? "Running" : "Idle"} + Status:{" "} + {status.state === "error" + ? ( + + Error{lastMacroStatus.message ? `: ${lastMacroStatus.message}` : ""} + + ) + : running + ? ( + + Running{lastMacroStatus.message ? ` (${lastMacroStatus.message})` : ""} + + ) + : "Idle"}
Agent Connection: {agentConnection ? "Connected" : "Not Connected"} +
+ Target Window:{" "} + {selectedWindow + ? `${selectedWindow.title} (${selectedWindow.handle})` + : data?.window_handle + ? `Handle: ${data.window_handle}` + : Not set} +
+ Mode: {data?.operation_mode || DEFAULT_OPERATION_MODE} +
+ Macro Type: {data?.macro_type || "keypress"} +
+ +
+ + {lastMacroStatus.timestamp + ? `Last event: ${new Date(lastMacroStatus.timestamp).toLocaleTimeString()}` + : ""} + +
+ + {/* Show available windows for debug (hidden from sidebar, but helpful for quick dropdown) */} +
+ Windows:{" "} + {windowList.length === 0 + ? No windows detected. + : windowList.map((w) => + + setNodes(nds => + nds.map(n => + n.id === id + ? { ...n, data: { ...n.data, window_handle: w.handle } } + : n + ) + ) + } + title={w.title} + > + {w.title} + + )}
); @@ -111,17 +309,18 @@ Supports manual, continuous, trigger, and one-shot modes for automation and even content: "Send Key Press or Typed Text to Window via Agent", component: MacroKeyPressNode, config: [ - { key: "window_handle", label: "Target Window", type: "text", defaultValue: "" }, - { key: "macro_type", label: "Macro Type", type: "select", options: ["keypress", "typed_text"], defaultValue: "keypress" }, - { key: "key", label: "Key", type: "text", defaultValue: "" }, - { key: "text", label: "Typed Text", type: "text", defaultValue: "" }, - { key: "interval_ms", label: "Interval (ms)", type: "text", defaultValue: "1000" }, - { key: "randomize_interval", label: "Randomize Interval", type: "select", options: ["true", "false"], defaultValue: "false" }, - { key: "random_min", label: "Random Min (ms)", type: "text", defaultValue: "750" }, - { key: "random_max", label: "Random Max (ms)", type: "text", defaultValue: "950" }, - { key: "operation_mode", label: "Operation Mode", type: "select", options: OPERATION_MODES, defaultValue: "Continuous" }, - { key: "active", label: "Macro Enabled", type: "select", options: ["true", "false"], defaultValue: "false" } -], + { key: "window_handle", label: "Target Window", type: "select", dynamicOptions: true, defaultValue: "" }, + { key: "macro_type", label: "Macro Type", type: "select", options: ["keypress", "typed_text"], defaultValue: "keypress" }, + { key: "key", label: "Key", type: "text", defaultValue: "" }, + { key: "text", label: "Typed Text", type: "text", defaultValue: "" }, + { key: "interval_ms", label: "Interval (ms)", type: "text", defaultValue: "1000" }, + { key: "randomize_interval", label: "Randomize Interval", type: "select", options: ["true", "false"], defaultValue: "false" }, + { key: "random_min", label: "Random Min (ms)", type: "text", defaultValue: "750" }, + { key: "random_max", label: "Random Max (ms)", type: "text", defaultValue: "950" }, + { key: "operation_mode", label: "Operation Mode", type: "select", options: OPERATION_MODES, defaultValue: "Continuous" }, + { key: "active", label: "Macro Enabled", type: "select", options: ["true", "false"], defaultValue: "false" }, + { key: "trigger", label: "Trigger Value", type: "text", defaultValue: "0" } + ], usage_documentation: ` ### Agent Role: Macro @@ -141,6 +340,9 @@ Supports manual, continuous, trigger, and one-shot modes for automation and even **Event-Driven Support:** - Chain with other Borealis nodes (text recognition, event triggers, etc). +**Live Status:** +- Displays last agent macro event and error feedback in node. + --- `.trim() }; diff --git a/Data/Server/server.py b/Data/Server/server.py index 5c6da2b..a0b4704 100644 --- a/Data/Server/server.py +++ b/Data/Server/server.py @@ -256,6 +256,40 @@ def receive_screenshot(data): def on_disconnect(): print("[WebSocket] Connection Disconnected") +# Macro Websocket Handlers +@socketio.on("macro_status") +def receive_macro_status(data): + """ + Receives macro status/errors from agent and relays to all clients + Expected payload: { + "agent_id": ..., + "node_id": ..., + "success": True/False, + "message": "...", + "timestamp": ... + } + """ + print(f"[Macro Status] Agent {data.get('agent_id')} Node {data.get('node_id')} Success: {data.get('success')} Msg: {data.get('message')}") + emit("macro_status", data, broadcast=True) + +@socketio.on("list_agent_windows") +def handle_list_agent_windows(data): + """ + Forwards list_agent_windows event to all agents (or filter for a specific agent_id). + """ + agent_id = data.get("agent_id") + # You can target a specific agent if you track rooms/sessions. + # For now, broadcast to all agents so the correct one can reply. + emit("list_agent_windows", data, broadcast=True) + +@socketio.on("agent_window_list") +def handle_agent_window_list(data): + """ + Relay the list of windows from the agent back to all connected clients. + """ + emit("agent_window_list", data, broadcast=True) + + # --------------------------------------------- # Server Launch # ---------------------------------------------