Implemented Live Macro Window Detection

Data/Agent/borealis-agent.py

@@ -10,6 +10,7 @@ from functools import partial
 from io import BytesIO
 import base64
 import traceback
+import random # Macro Randomization
 import platform # OS Detection
 import importlib.util
 
@@ -338,7 +339,6 @@ class ScreenshotRegion(QtWidgets.QWidget):
 async def screenshot_task(cfg):
     nid=cfg.get('node_id')
     alias=cfg.get('alias','')
-    print(f"[DEBUG] Running screenshot_task for {nid}")
     r=CONFIG.data['regions'].get(nid)
     if r:
         region=(r['x'],r['y'],r['w'],r['h'])
@@ -369,40 +369,116 @@ async def screenshot_task(cfg):
 
 # ---------------- Macro Task ----------------
 async def macro_task(cfg):
+    """
+    Improved macro_task supporting all operation modes, live config, error reporting, and UI feedback.
+    """
     nid = cfg.get('node_id')
-    window_handle = cfg.get('window_handle')
-    mode = cfg.get('operation_mode', 'keypress')  # 'keypress' or 'typed_text'
-    key = cfg.get('key')
-    text = cfg.get('text')
-    interval_ms = int(cfg.get('interval_ms', 1000))
-    randomize = cfg.get('randomize_interval', False)
-    random_min = int(cfg.get('random_min', 750))
-    random_max = int(cfg.get('random_max', 950))
-    active = cfg.get('active', True)  # Whether macro is "started" or "paused"
-    print(f"[DEBUG] Macro task for node {nid} started on window {window_handle}")
 
-    import random
+    # Track trigger state for edge/level changes
-    try:
+    last_trigger_value = 0
-        while True:
+    has_run_once = False
-            if not active:
+
-                await asyncio.sleep(0.2)
+    while True:
-                continue
+        # Always re-fetch config (hot reload support)
-            if mode == 'keypress' and key:
+        # (In reality, you might want to deep-copy or re-provision on config update, but for MVP we refetch each tick)
-                macro_engines.send_keypress_to_window(window_handle, key)
+        window_handle = cfg.get('window_handle')
-            elif mode == 'typed_text' and text:
+        macro_type = cfg.get('macro_type', 'keypress')  # Now matches UI config
-                macro_engines.type_text_to_window(window_handle, text)
+        operation_mode = cfg.get('operation_mode', 'Continuous')
-            # Interval logic
+        key = cfg.get('key')
-            if randomize:
+        text = cfg.get('text')
-                ms = random.randint(random_min, random_max)
+        interval_ms = int(cfg.get('interval_ms', 1000))
+        randomize = cfg.get('randomize_interval', False)
+        random_min = int(cfg.get('random_min', 750))
+        random_max = int(cfg.get('random_max', 950))
+        active = cfg.get('active', True)
+        trigger = int(cfg.get('trigger', 0))  # For trigger modes; default 0 if not set
+
+        # Define helper for error reporting
+        async def emit_macro_status(success, message=""):
+            await sio.emit('macro_status', {
+                "agent_id": AGENT_ID,
+                "node_id": nid,
+                "success": success,
+                "message": message,
+                "timestamp": int(asyncio.get_event_loop().time() * 1000)
+            })
+
+        # Stopped state (paused from UI)
+        if not (active is True or str(active).lower() == "true"):
+            await asyncio.sleep(0.2)
+            continue
+
+        try:
+            send_macro = False
+
+            # Operation Mode Logic
+            if operation_mode == "Run Once":
+                if not has_run_once:
+                    send_macro = True
+                    has_run_once = True  # Only run once, then stop
+            elif operation_mode == "Continuous":
+                send_macro = True  # Always run every interval
+            elif operation_mode == "Trigger-Continuous":
+                # Only run while trigger is "1"
+                if trigger == 1:
+                    send_macro = True
+                else:
+                    send_macro = False
+            elif operation_mode == "Trigger-Once":
+                # Run only on rising edge: 0->1
+                if last_trigger_value == 0 and trigger == 1:
+                    send_macro = True
+                else:
+                    send_macro = False
+                last_trigger_value = trigger
             else:
-                ms = interval_ms
+                # Unknown mode: default to "Continuous"
-            await asyncio.sleep(ms / 1000.0)
+                send_macro = True
-    except asyncio.CancelledError:
+
-        print(f"[TASK] Macro role {nid} cancelled.")
+            if send_macro:
-    except Exception as e:
+                # Actually perform macro
-        print(f"[ERROR] Macro task {nid} failed: {e}")
+                if macro_type == 'keypress' and key:
-        import traceback
+                    result = macro_engines.send_keypress_to_window(window_handle, key)
-        traceback.print_exc()
+                elif macro_type == 'typed_text' and text:
+                    result = macro_engines.type_text_to_window(window_handle, text)
+                else:
+                    await emit_macro_status(False, "Invalid macro type or missing key/text")
+                    await asyncio.sleep(0.2)
+                    continue
+
+                # Result may be True or (False, error)
+                if isinstance(result, tuple):
+                    success, err = result
+                else:
+                    success, err = bool(result), ""
+
+                if success:
+                    await emit_macro_status(True, f"Macro sent: {macro_type}")
+                else:
+                    await emit_macro_status(False, err or "Unknown macro engine failure")
+            else:
+                # No macro to send this cycle, just idle
+                await asyncio.sleep(0.05)
+
+            # Timing: only wait if we did send macro this tick
+            if send_macro:
+                if randomize:
+                    ms = random.randint(random_min, random_max)
+                else:
+                    ms = interval_ms
+                await asyncio.sleep(ms / 1000.0)
+            else:
+                await asyncio.sleep(0.1)  # No macro action: check again soon
+
+        except asyncio.CancelledError:
+            print(f"[TASK] Macro role {nid} cancelled.")
+            break
+        except Exception as e:
+            print(f"[ERROR] Macro task {nid} failed: {e}")
+            import traceback
+            traceback.print_exc()
+            await emit_macro_status(False, str(e))
+            await asyncio.sleep(0.5)
 
 # ---------------- Config Watcher ----------------
 async def config_watcher():

Data/Server/WebUI/src/Node_Configuration_Sidebar.jsx

@@ -43,13 +43,27 @@ export default function NodeConfigurationSidebar({ drawerOpen, setDrawerOpen, ti
     return config.map((field, index) => {
       const value = nodeData?.[field.key] || "";
 
-      return (
+      // ---- DYNAMIC DROPDOWN SUPPORT ----
-        <Box key={index} sx={{ mb: 2 }}>
+      if (field.type === "select") {
-          <Typography variant="body2" sx={{ color: "#ccc", mb: 0.5 }}>
+        let options = field.options || [];
-            {field.label || field.key}
-          </Typography>
 
-          {field.type === "select" ? (
+        // Handle dynamic options for things like Target Window
+        if (field.dynamicOptions && nodeData?.windowList && Array.isArray(nodeData.windowList)) {
+          options = nodeData.windowList
+            .map(win => ({
+              value: String(win.handle),
+              label: `${win.title} (${win.handle})`
+            }))
+            .sort((a, b) => a.label.localeCompare(b.label, undefined, { sensitivity: "base" }));
+        } else {
+          options = options.map(opt => ({ value: opt, label: opt }));
+        }
+
+        return (
+          <Box key={index} sx={{ mb: 2 }}>
+            <Typography variant="body2" sx={{ color: "#ccc", mb: 0.5 }}>
+              {field.label || field.key}
+            </Typography>
             <TextField
               select
               fullWidth
@@ -112,42 +126,57 @@ export default function NodeConfigurationSidebar({ drawerOpen, setDrawerOpen, ti
                 }
               }}
             >
-              {(field.options || []).map((opt, idx) => (
+              {options.length === 0 ? (
-                <MenuItem key={idx} value={opt}>
+                <MenuItem disabled value="">
-                  {opt}
+                  {field.label === "Target Window"
+                    ? "No windows detected"
+                    : "No options"}
                 </MenuItem>
-              ))}
+              ) : (
+                options.map((opt, idx) => (
+                  <MenuItem key={idx} value={opt.value}>
+                    {opt.label}
+                  </MenuItem>
+                ))
+              )}
             </TextField>
+          </Box>
+        );
+      }
+      // ---- END DYNAMIC DROPDOWN SUPPORT ----
 
-          ) : (
+      return (
-            <TextField
+        <Box key={index} sx={{ mb: 2 }}>
-              variant="outlined"
+          <Typography variant="body2" sx={{ color: "#ccc", mb: 0.5 }}>
-              size="small"
+            {field.label || field.key}
-              fullWidth
+          </Typography>
-              value={value}
+          <TextField
-              onChange={(e) => {
+            variant="outlined"
-                const newValue = e.target.value;
+            size="small"
-                if (!nodeId) return;
+            fullWidth
-                effectiveSetNodes((nds) =>
+            value={value}
-                  nds.map((n) =>
+            onChange={(e) => {
-                    n.id === nodeId
+              const newValue = e.target.value;
-                      ? { ...n, data: { ...n.data, [field.key]: newValue } }
+              if (!nodeId) return;
-                      : n
+              effectiveSetNodes((nds) =>
-                  )
+                nds.map((n) =>
-                );
+                  n.id === nodeId
-                window.BorealisValueBus[nodeId] = newValue;
+                    ? { ...n, data: { ...n.data, [field.key]: newValue } }
-              }}
+                    : n
-              InputProps={{
+                )
-                sx: {
+              );
-                  backgroundColor: "#1e1e1e",
+              window.BorealisValueBus[nodeId] = newValue;
-                  color: "#ccc",
+            }}
-                  "& fieldset": { borderColor: "#444" },
+            InputProps={{
-                  "&:hover fieldset": { borderColor: "#666" },
+              sx: {
-                  "&.Mui-focused fieldset": { borderColor: "#58a6ff" }
+                backgroundColor: "#1e1e1e",
-                }
+                color: "#ccc",
-              }}
+                "& fieldset": { borderColor: "#444" },
-            />
+                "&:hover fieldset": { borderColor: "#666" },
-          )}
+                "&.Mui-focused fieldset": { borderColor: "#58a6ff" }
+              }
+            }}
+          />
         </Box>
       );
     });

Data/Server/WebUI/src/nodes/Agent Roles/Node_Agent_Role_Macro.jsx

@@ -17,21 +17,130 @@ const OPERATION_MODES = [
   "Trigger-Continuous"
 ];
 
+const MACRO_TYPES = [
+  "keypress",
+  "typed_text"
+];
+
+const statusColors = {
+  idle: "#333",
+  running: "#00d18c",
+  error: "#ff4f4f",
+  success: "#00d18c"
+};
+
 const MacroKeyPressNode = ({ id, data }) => {
   const { setNodes, getNodes } = useReactFlow();
   const edges = useStore((state) => state.edges);
+  const [windowList, setWindowList] = useState([]);
+  const [status, setStatus] = useState({ state: "idle", message: "" });
+  const socketRef = useRef(null);
 
   // Determine if agent is connected
   const agentEdge = edges.find((e) => e.target === id && e.targetHandle === "agent");
   const agentNode = agentEdge && getNodes().find((n) => n.id === agentEdge.source);
   const agentConnection = !!(agentNode && agentNode.data && agentNode.data.agent_id);
+  const agent_id = agentNode && agentNode.data && agentNode.data.agent_id;
 
   // Macro run/trigger state (sidebar sets this via config, but node UI just shows status)
   const running = data?.active === true || data?.active === "true";
 
-  // Node UI (no config fields, only status)
+  // Store for last macro error/status
+  const [lastMacroStatus, setLastMacroStatus] = useState({ success: true, message: "", timestamp: null });
+
+  // Setup WebSocket for agent macro status updates
+  useEffect(() => {
+    if (!window.BorealisSocket) return;
+    const socket = window.BorealisSocket;
+    socketRef.current = socket;
+
+    function handleMacroStatus(payload) {
+      if (
+        payload &&
+        payload.agent_id === agent_id &&
+        payload.node_id === id
+      ) {
+        setLastMacroStatus({
+          success: !!payload.success,
+          message: payload.message || "",
+          timestamp: payload.timestamp || Date.now()
+        });
+        setStatus({
+          state: payload.success ? "success" : "error",
+          message: payload.message || (payload.success ? "Success" : "Error")
+        });
+      }
+    }
+
+    socket.on("macro_status", handleMacroStatus);
+    return () => {
+      socket.off("macro_status", handleMacroStatus);
+    };
+  }, [agent_id, id]);
+
+  // Auto-refresh window list from agent
+  useEffect(() => {
+    let intervalId = null;
+    async function fetchWindows() {
+      if (window.BorealisSocket && agentConnection) {
+        window.BorealisSocket.emit("list_agent_windows", {
+          agent_id
+        });
+      }
+    }
+    fetchWindows();
+    intervalId = setInterval(fetchWindows, WINDOW_LIST_REFRESH_MS);
+
+    // Listen for agent_window_list updates
+    function handleAgentWindowList(payload) {
+      if (payload?.agent_id === agent_id && Array.isArray(payload.windows)) {
+        setWindowList(payload.windows);
+
+        // Store windowList in node data for sidebar dynamic dropdowns
+        setNodes(nds =>
+          nds.map(n =>
+            n.id === id
+              ? { ...n, data: { ...n.data, windowList: payload.windows } }
+              : n
+          )
+        );
+      }
+    }
+    if (window.BorealisSocket) {
+      window.BorealisSocket.on("agent_window_list", handleAgentWindowList);
+    }
+
+    return () => {
+      clearInterval(intervalId);
+      if (window.BorealisSocket) {
+        window.BorealisSocket.off("agent_window_list", handleAgentWindowList);
+      }
+    };
+  }, [agent_id, agentConnection, setNodes, id]);
+
+  // UI: Start/Pause Button
+  const handleToggleMacro = () => {
+    setNodes(nds =>
+      nds.map(n =>
+        n.id === id
+          ? {
+              ...n,
+              data: {
+                ...n.data,
+                active: n.data?.active === true || n.data?.active === "true" ? "false" : "true"
+              }
+            }
+          : n
+      )
+    );
+  };
+
+  // Optional: Show which window is targeted by name
+  const selectedWindow = (windowList || []).find(w => String(w.handle) === String(data?.window_handle));
+
+  // Node UI (no config fields, only status + window list)
   return (
-    <div className="borealis-node" style={{ minWidth: 240, position: "relative" }}>
+    <div className="borealis-node" style={{ minWidth: 280, position: "relative" }}>
       {/* --- INPUT LABELS & HANDLES --- */}
       <div style={{
         position: "absolute",
@@ -85,16 +194,105 @@ const MacroKeyPressNode = ({ id, data }) => {
             transform: "translateY(-50%)",
             height: "10px",
             borderRadius: "50%",
-            backgroundColor: running ? "#00d18c" : "#333",
+            backgroundColor:
+              status.state === "error"
+                ? statusColors.error
+                : running
+                ? statusColors.running
+                : statusColors.idle,
             border: "1px solid #222"
           }}
         />
       </div>
 
       <div className="borealis-node-content">
-        <strong>Status</strong>: {running ? "Running" : "Idle"}
+        <strong>Status</strong>:{" "}
+        {status.state === "error"
+          ? (
+            <span style={{ color: "#ff4f4f" }}>
+              Error{lastMacroStatus.message ? `: ${lastMacroStatus.message}` : ""}
+            </span>
+          )
+          : running
+            ? (
+              <span style={{ color: "#00d18c" }}>
+                Running{lastMacroStatus.message ? ` (${lastMacroStatus.message})` : ""}
+              </span>
+            )
+            : "Idle"}
         <br />
         <strong>Agent Connection</strong>: {agentConnection ? "Connected" : "Not Connected"}
+        <br />
+        <strong>Target Window</strong>:{" "}
+        {selectedWindow
+          ? `${selectedWindow.title} (${selectedWindow.handle})`
+          : data?.window_handle
+            ? `Handle: ${data.window_handle}`
+            : <span style={{ color: "#888" }}>Not set</span>}
+        <br />
+        <strong>Mode</strong>: {data?.operation_mode || DEFAULT_OPERATION_MODE}
+        <br />
+        <strong>Macro Type</strong>: {data?.macro_type || "keypress"}
+        <br />
+        <button
+          onClick={handleToggleMacro}
+          style={{
+            marginTop: 8,
+            padding: "4px 10px",
+            background: running ? "#3a3a3a" : "#0475c2",
+            color: running ? "#fff" : "#fff",
+            border: "1px solid #0475c2",
+            borderRadius: 3,
+            fontSize: "11px",
+            cursor: "pointer"
+          }}
+        >
+          {running ? "Pause Macro" : "Start Macro"}
+        </button>
+        <br />
+        <span style={{ fontSize: "9px", color: "#aaa" }}>
+          {lastMacroStatus.timestamp
+            ? `Last event: ${new Date(lastMacroStatus.timestamp).toLocaleTimeString()}`
+            : ""}
+        </span>
+      </div>
+
+      {/* Show available windows for debug (hidden from sidebar, but helpful for quick dropdown) */}
+      <div style={{ marginTop: 8, fontSize: "9px", color: "#58a6ff" }}>
+        <b>Windows:</b>{" "}
+        {windowList.length === 0
+          ? <span style={{ color: "#999" }}>No windows detected.</span>
+          : windowList.map((w) =>
+              <span
+                key={w.handle}
+                style={{
+                  background:
+                    String(w.handle) === String(data?.window_handle)
+                      ? "#333"
+                      : "transparent",
+                  borderRadius: 3,
+                  padding: "1px 4px",
+                  marginRight: 4,
+                  border:
+                    String(w.handle) === String(data?.window_handle)
+                      ? "1px solid #58a6ff"
+                      : "1px solid #222",
+                  cursor: "pointer"
+                }}
+                onClick={() =>
+                  setNodes(nds =>
+                    nds.map(n =>
+                      n.id === id
+                        ? { ...n, data: { ...n.data, window_handle: w.handle } }
+                        : n
+                    )
+                  )
+                }
+                title={w.title}
+              >
+                {w.title}
+              </span>
+            )}
       </div>
     </div>
   );
@@ -111,17 +309,18 @@ Supports manual, continuous, trigger, and one-shot modes for automation and even
   content: "Send Key Press or Typed Text to Window via Agent",
   component: MacroKeyPressNode,
   config: [
-  { key: "window_handle", label: "Target Window", type: "text", defaultValue: "" },
+    { key: "window_handle", label: "Target Window", type: "select", dynamicOptions: true, defaultValue: "" },
-  { key: "macro_type", label: "Macro Type", type: "select", options: ["keypress", "typed_text"], defaultValue: "keypress" },
+    { key: "macro_type", label: "Macro Type", type: "select", options: ["keypress", "typed_text"], defaultValue: "keypress" },
-  { key: "key", label: "Key", type: "text", defaultValue: "" },
+    { key: "key", label: "Key", type: "text", defaultValue: "" },
-  { key: "text", label: "Typed Text", type: "text", defaultValue: "" },
+    { key: "text", label: "Typed Text", type: "text", defaultValue: "" },
-  { key: "interval_ms", label: "Interval (ms)", type: "text", defaultValue: "1000" },
+    { key: "interval_ms", label: "Interval (ms)", type: "text", defaultValue: "1000" },
-  { key: "randomize_interval", label: "Randomize Interval", type: "select", options: ["true", "false"], defaultValue: "false" },
+    { key: "randomize_interval", label: "Randomize Interval", type: "select", options: ["true", "false"], defaultValue: "false" },
-  { key: "random_min", label: "Random Min (ms)", type: "text", defaultValue: "750" },
+    { key: "random_min", label: "Random Min (ms)", type: "text", defaultValue: "750" },
-  { key: "random_max", label: "Random Max (ms)", type: "text", defaultValue: "950" },
+    { key: "random_max", label: "Random Max (ms)", type: "text", defaultValue: "950" },
-  { key: "operation_mode", label: "Operation Mode", type: "select", options: OPERATION_MODES, defaultValue: "Continuous" },
+    { key: "operation_mode", label: "Operation Mode", type: "select", options: OPERATION_MODES, defaultValue: "Continuous" },
-  { key: "active", label: "Macro Enabled", type: "select", options: ["true", "false"], defaultValue: "false" }
+    { key: "active", label: "Macro Enabled", type: "select", options: ["true", "false"], defaultValue: "false" },
-],
+    { key: "trigger", label: "Trigger Value", type: "text", defaultValue: "0" }
+  ],
   usage_documentation: `
 ### Agent Role: Macro
 
@@ -141,6 +340,9 @@ Supports manual, continuous, trigger, and one-shot modes for automation and even
 **Event-Driven Support:**
 - Chain with other Borealis nodes (text recognition, event triggers, etc).
 
+**Live Status:**
+- Displays last agent macro event and error feedback in node.
+
 ---
   `.trim()
 };

Data/Server/server.py

@@ -256,6 +256,40 @@ def receive_screenshot(data):
 def on_disconnect():
     print("[WebSocket] Connection Disconnected")
 
+# Macro Websocket Handlers
+@socketio.on("macro_status")
+def receive_macro_status(data):
+    """
+    Receives macro status/errors from agent and relays to all clients
+    Expected payload: {
+        "agent_id": ...,
+        "node_id": ...,
+        "success": True/False,
+        "message": "...",
+        "timestamp": ...
+    }
+    """
+    print(f"[Macro Status] Agent {data.get('agent_id')} Node {data.get('node_id')} Success: {data.get('success')} Msg: {data.get('message')}")
+    emit("macro_status", data, broadcast=True)
+
+@socketio.on("list_agent_windows")
+def handle_list_agent_windows(data):
+    """
+    Forwards list_agent_windows event to all agents (or filter for a specific agent_id).
+    """
+    agent_id = data.get("agent_id")
+    # You can target a specific agent if you track rooms/sessions.
+    # For now, broadcast to all agents so the correct one can reply.
+    emit("list_agent_windows", data, broadcast=True)
+
+@socketio.on("agent_window_list")
+def handle_agent_window_list(data):
+    """
+    Relay the list of windows from the agent back to all connected clients.
+    """
+    emit("agent_window_list", data, broadcast=True)
+
+
 # ---------------------------------------------
 # Server Launch
 # ---------------------------------------------