From 47d13c017815f1e1cfc36860d924949949c24873 Mon Sep 17 00:00:00 2001 From: Nicole Rappe Date: Sun, 1 Jun 2025 13:29:29 -0600 Subject: [PATCH] Initial Macro Node Rework (Testing Phase) --- .../WebUI/src/nodes/Automation/Node_Macro.jsx | 509 ++++++++++++++++++ .../nodes/Automation/Node_Macro_KeyPress.jsx | 296 ---------- 2 files changed, 509 insertions(+), 296 deletions(-) create mode 100644 Data/Server/WebUI/src/nodes/Automation/Node_Macro.jsx delete mode 100644 Data/Server/WebUI/src/nodes/Automation/Node_Macro_KeyPress.jsx diff --git a/Data/Server/WebUI/src/nodes/Automation/Node_Macro.jsx b/Data/Server/WebUI/src/nodes/Automation/Node_Macro.jsx new file mode 100644 index 0000000..1a90e51 --- /dev/null +++ b/Data/Server/WebUI/src/nodes/Automation/Node_Macro.jsx @@ -0,0 +1,509 @@ +////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: /Data/WebUI/src/nodes/Automation/Node_Macro.jsx +import React, { useState, useEffect, useRef } from "react"; +import { Handle, Position, useReactFlow, useStore } from "reactflow"; +import Keyboard from "react-simple-keyboard"; +import "react-simple-keyboard/build/css/index.css"; + +// Default update interval for window list refresh (in ms) +const WINDOW_LIST_REFRESH_MS = 4000; + +if (!window.BorealisValueBus) window.BorealisValueBus = {}; +if (!window.BorealisUpdateRate) window.BorealisUpdateRate = 100; + +const DEFAULT_OPERATION_MODE = "Continuous"; +const OPERATION_MODES = [ + "Continuous", + "Trigger-Continuous", + "Trigger-Once", + "Run Once" +]; + +const MACRO_TYPES = [ + { value: "keypress", label: "Single Keypress" }, + { value: "typed_text", label: "Typed Text" } +]; + +const MacroKeyPressNode = ({ id, data }) => { + const { setNodes } = useReactFlow(); + const edges = useStore((state) => state.edges); + + // State for agent window list dropdown + const [windowList, setWindowList] = useState([]); + const [selectedWindow, setSelectedWindow] = useState(data?.window_handle || ""); + const [windowListStatus, setWindowListStatus] = useState("Loading..."); + + // State for keyboard/text settings + const [macroType, setMacroType] = useState(data?.macro_type || "keypress"); + const [keyPressed, setKeyPressed] = useState(data?.key || ""); + const [typedText, setTypedText] = useState(data?.text || ""); + const [showKeyboard, setShowKeyboard] = useState(false); + const [layoutName, setLayoutName] = useState("default"); + + // Interval/randomization settings + const [intervalMs, setIntervalMs] = useState(data?.interval_ms || 1000); + const [randomize, setRandomize] = useState(!!data?.randomize_interval); + const [randomMin, setRandomMin] = useState(data?.random_min || 750); + const [randomMax, setRandomMax] = useState(data?.random_max || 950); + + // Macro run/trigger state + const [operationMode, setOperationMode] = useState(data?.operation_mode || DEFAULT_OPERATION_MODE); + const [running, setRunning] = useState(data?.active ?? false); + + // For Trigger-Once logic + const triggerActiveRef = useRef(false); + + // Fetch windows from agent using WebSocket + useEffect(() => { + let isMounted = true; + let interval = null; + + const fetchWindowList = () => { + setWindowListStatus("Loading..."); + if (!window.BorealisSocket) return setWindowListStatus("No agent connection"); + // Find the upstream agent node, get its agent_id + let agentId = null; + for (const e of edges) { + if (e.target === id && e.sourceHandle === "provisioner") { + const agentNode = window.BorealisFlowNodes?.find((n) => n.id === e.source); + agentId = agentNode?.data?.agent_id; + } + } + if (!agentId) return setWindowListStatus("No agent connected"); + window.BorealisSocket.emit("list_agent_windows", { agent_id: agentId }); + }; + + // Listen for reply + function handleAgentWindowList(payload) { + if (!isMounted) return; + if (!payload || !payload.windows) return setWindowListStatus("No windows found"); + setWindowList(payload.windows); + setWindowListStatus(payload.windows.length ? "" : "No windows found"); + } + if (window.BorealisSocket) { + window.BorealisSocket.on("agent_window_list", handleAgentWindowList); + } + fetchWindowList(); + interval = setInterval(fetchWindowList, WINDOW_LIST_REFRESH_MS); + + return () => { + isMounted = false; + if (window.BorealisSocket) { + window.BorealisSocket.off("agent_window_list", handleAgentWindowList); + } + clearInterval(interval); + }; + }, [id, edges]); + + // Macro state (simulate agent push) + useEffect(() => { + setNodes((nds) => + nds.map((n) => + n.id === id + ? { + ...n, + data: { + ...n.data, + window_handle: selectedWindow, + macro_type: macroType, + key: keyPressed, + text: typedText, + interval_ms: intervalMs, + randomize_interval: randomize, + random_min: randomMin, + random_max: randomMax, + operation_mode: operationMode, + active: running + } + } + : n + ) + ); + // BorealisValueBus: Set running status as "1" or "0" + window.BorealisValueBus[id] = running ? "1" : "0"; + }, [ + id, + setNodes, + selectedWindow, + macroType, + keyPressed, + typedText, + intervalMs, + randomize, + randomMin, + randomMax, + operationMode, + running + ]); + + // Input trigger/handle logic for trigger-based modes + useEffect(() => { + if (!["Trigger-Continuous", "Trigger-Once"].includes(operationMode)) return; + // Find first left input edge + const edge = edges.find((e) => e.target === id); + if (!edge) return; + const upstreamValue = window.BorealisValueBus[edge.source]; + if (operationMode === "Trigger-Continuous") { + setRunning(upstreamValue === "1"); + } else if (operationMode === "Trigger-Once") { + // Only fire once per rising edge + if (upstreamValue === "1" && !triggerActiveRef.current) { + setRunning(true); + triggerActiveRef.current = true; + setTimeout(() => setRunning(false), 10); // Simulate a quick one-shot macro + } else if (upstreamValue !== "1" && triggerActiveRef.current) { + triggerActiveRef.current = false; + } + } + }, [edges, id, operationMode]); + + // Handle Start/Stop button for manual modes + const handleStartStop = () => { + setRunning((v) => !v); + }; + + // Keyboard overlay logic + const onKeyPress = (button) => { + // SHIFT or CAPS toggling: + if (button === "{shift}" || button === "{lock}") { + setLayoutName((prev) => (prev === "default" ? "shift" : "default")); + return; + } + // Accept only standard keys (not function/meta keys) + const skipKeys = [ + "{bksp}", "{space}", "{tab}", "{enter}", "{escape}", + "{f1}", "{f2}", "{f3}", "{f4}", "{f5}", "{f6}", + "{f7}", "{f8}", "{f9}", "{f10}", "{f11}", "{f12}", + "{shift}", "{lock}" + ]; + if (!skipKeys.includes(button)) { + setKeyPressed(button); + setShowKeyboard(false); + } + }; + + // Node UI + return ( +
+ + + +
+ Agent Role: Macro +
+
+ +
+
+ Sends macro input to selected window via agent. +
+ + +
+ {windowListStatus} +
+ + {/* Macro Type */} + + + + {/* Key or Text Selection */} + {macroType === "keypress" ? ( + <> + +
+ + +
+ + ) : ( + <> + + setTypedText(e.target.value)} + style={{ ...inputStyle, width: "100%" }} + maxLength={256} + /> + + )} + + {/* Interval */} + + setIntervalMs(Number(e.target.value))} + disabled={randomize} + style={{ + ...inputStyle, + backgroundColor: randomize ? "#2a2a2a" : "#1e1e1e" + }} + /> + + {/* Randomize Interval */} + + {randomize && ( +
+ setRandomMin(Number(e.target.value))} + style={{ ...inputStyle, flex: 1 }} + /> + setRandomMax(Number(e.target.value))} + style={{ ...inputStyle, flex: 1 }} + /> +
+ )} + + {/* Operation Mode */} + + + + {/* Start/Stop Button */} + {operationMode === "Continuous" || operationMode === "Run Once" ? ( + + ) : null} +
+ + {/* Keyboard Overlay */} + {showKeyboard && ( +
+
+
+ Full Keyboard +
+ ? {shift}", + "{space}" + ] + }} + display={{ + "{bksp}": "⌫", + "{escape}": "esc", + "{tab}": "tab", + "{lock}": "caps", + "{enter}": "enter", + "{shift}": "shift", + "{space}": "space", + "{f1}": "F1", + "{f2}": "F2", + "{f3}": "F3", + "{f4}": "F4", + "{f5}": "F5", + "{f6}": "F6", + "{f7}": "F7", + "{f8}": "F8", + "{f9}": "F9", + "{f10}": "F10", + "{f11}": "F11", + "{f12}": "F12" + }} + /> +
+ +
+
+
+ )} +
+ ); +}; + +// ----- Node Catalog Export ----- +const inputStyle = { + width: "100%", + fontSize: "9px", + padding: "4px", + color: "#ccc", + border: "1px solid #444", + borderRadius: "2px", + marginBottom: "6px" +}; + +const buttonStyle = { + fontSize: "9px", + padding: "4px 8px", + backgroundColor: "#1e1e1e", + color: "#ccc", + border: "1px solid #444", + borderRadius: "2px", + cursor: "pointer" +}; + +const keyboardOverlay = { + position: "fixed", + top: 0, + left: 0, + width: "100vw", + height: "100vh", + zIndex: 1000, + backgroundColor: "rgba(0, 0, 0, 0.8)", + display: "flex", + justifyContent: "center", + alignItems: "center" +}; + +const keyboardContainer = { + backgroundColor: "#1e1e1e", + padding: "16px", + borderRadius: "6px", + border: "1px solid #444", + zIndex: 1001, + maxWidth: "650px" +}; + +export default { + type: "Macro_KeyPress", + label: "Agent Role: Macro", + description: ` +Send automated key presses or typed text to any open application window on the connected agent. +Supports manual, continuous, trigger, and one-shot modes for automation and event-driven workflows. +`, + content: "Send Key Press or Typed Text to Window via Agent", + component: MacroKeyPressNode, + config: [ + { key: "window_handle", label: "Target Window", type: "text" }, + { key: "macro_type", label: "Macro Type", type: "select", options: ["keypress", "typed_text"] }, + { key: "key", label: "Key", type: "text" }, + { key: "text", label: "Typed Text", type: "text" }, + { key: "interval_ms", label: "Interval (ms)", type: "text" }, + { key: "randomize_interval", label: "Randomize Interval", type: "select", options: ["true", "false"] }, + { key: "random_min", label: "Random Min (ms)", type: "text" }, + { key: "random_max", label: "Random Max (ms)", type: "text" }, + { key: "operation_mode", label: "Operation Mode", type: "select", options: OPERATION_MODES }, + { key: "active", label: "Active", type: "select", options: ["true", "false"] } + ], + usage_documentation: ` +### Agent Role: Macro + +**Modes:** +- **Continuous**: Macro sends input non-stop when started by button. +- **Trigger-Continuous**: Macro sends input as long as upstream trigger is "1". +- **Trigger-Once**: Macro fires once per upstream "1" (one-shot edge). +- **Run Once**: Macro runs only once when started by button. + +**Macro Types:** +- **Single Keypress**: Press a single key. +- **Typed Text**: Types out a string. + +**Window Target:** +- Dropdown of live windows from agent, stays updated. + +**Event-Driven Support:** +- Chain with other Borealis nodes (text recognition, event triggers, etc). + +--- + `.trim() +}; diff --git a/Data/Server/WebUI/src/nodes/Automation/Node_Macro_KeyPress.jsx b/Data/Server/WebUI/src/nodes/Automation/Node_Macro_KeyPress.jsx deleted file mode 100644 index e900cac..0000000 --- a/Data/Server/WebUI/src/nodes/Automation/Node_Macro_KeyPress.jsx +++ /dev/null @@ -1,296 +0,0 @@ -////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: /Data/WebUI/src/nodes/Automation/Node_Macro_KeyPress.jsx -import React, { useState, useRef } from "react"; -import { Handle, Position } from "reactflow"; -import Keyboard from "react-simple-keyboard"; -import "react-simple-keyboard/build/css/index.css"; - -if (!window.BorealisValueBus) window.BorealisValueBus = {}; -if (!window.BorealisUpdateRate) window.BorealisUpdateRate = 100; - -/** - * KeyPressNode: - * - Full keyboard with SHIFT toggling - * - Press F-keys, digits, letters, or symbols - * - Single key stored, overlay closes - * - SHIFT or CAPS toggles "default" <-> "shift" - */ - -const KeyPressNode = ({ id, data }) => { - const [selectedWindow, setSelectedWindow] = useState(data?.selectedWindow || ""); - const [keyPressed, setKeyPressed] = useState(data?.keyPressed || ""); - const [intervalMs, setIntervalMs] = useState(data?.intervalMs || 1000); - const [randomRangeEnabled, setRandomRangeEnabled] = useState(false); - const [randomMin, setRandomMin] = useState(750); - const [randomMax, setRandomMax] = useState(950); - - // Keyboard overlay - const [showKeyboard, setShowKeyboard] = useState(false); - const [layoutName, setLayoutName] = useState("default"); - - // A simple set of Windows for demonstration - const fakeWindows = ["Notepad", "Chrome", "Discord", "Visual Studio Code"]; - - // This function is triggered whenever the user taps a key on the virtual keyboard - const onKeyPress = (button) => { - // SHIFT or CAPS toggling: - if (button === "{shift}" || button === "{lock}") { - handleShift(); - return; - } - - // Example skip list: these won't be stored as final single key - const skipKeys = [ - "{bksp}", "{space}", "{tab}", "{enter}", "{escape}", - "{f1}", "{f2}", "{f3}", "{f4}", "{f5}", "{f6}", - "{f7}", "{f8}", "{f9}", "{f10}", "{f11}", "{f12}", - "{shift}", "{lock}" - ]; - - // If the pressed button is not in skipKeys, let's store it and close - if (!skipKeys.includes(button)) { - setKeyPressed(button); - setShowKeyboard(false); - } - }; - - // Toggle between "default" layout and "shift" layout - const handleShift = () => { - setLayoutName((prev) => (prev === "default" ? "shift" : "default")); - }; - - return ( -
- {/* React Flow Handles */} - - - - {/* Node Header */} -
- Key Press -
-
- - {/* Node Content */} -
-
- Sends keypress to selected window on trigger. -
- - {/* Window Selector */} - - - - {/* Key: "Select Key" button & readOnly input */} - -
- - -
- - {/* Interval Configuration */} - - setIntervalMs(Number(e.target.value))} - disabled={randomRangeEnabled} - style={{ - ...inputStyle, - backgroundColor: randomRangeEnabled ? "#2a2a2a" : "#1e1e1e" - }} - /> - - {/* Random Interval */} - - - {randomRangeEnabled && ( -
- setRandomMin(Number(e.target.value))} - style={{ ...inputStyle, flex: 1 }} - /> - setRandomMax(Number(e.target.value))} - style={{ ...inputStyle, flex: 1 }} - /> -
- )} -
- - {/* Keyboard Overlay */} - {showKeyboard && ( -
-
-
- Full Keyboard -
- ? {shift}", - "{space}" - ] - }} - display={{ - "{bksp}": "⌫", - "{escape}": "esc", - "{tab}": "tab", - "{lock}": "caps", - "{enter}": "enter", - "{shift}": "shift", - "{space}": "space", - "{f1}": "F1", - "{f2}": "F2", - "{f3}": "F3", - "{f4}": "F4", - "{f5}": "F5", - "{f6}": "F6", - "{f7}": "F7", - "{f8}": "F8", - "{f9}": "F9", - "{f10}": "F10", - "{f11}": "F11", - "{f12}": "F12" - }} - /> -
- -
-
-
- )} -
- ); -}; - -/* Basic styling objects */ -const inputStyle = { - width: "100%", - fontSize: "9px", - padding: "4px", - color: "#ccc", - border: "1px solid #444", - borderRadius: "2px", - marginBottom: "6px" -}; - -const buttonStyle = { - fontSize: "9px", - padding: "4px 8px", - backgroundColor: "#1e1e1e", - color: "#ccc", - border: "1px solid #444", - borderRadius: "2px", - cursor: "pointer" -}; - -const keyboardOverlay = { - position: "fixed", - top: 0, - left: 0, - width: "100vw", - height: "100vh", - zIndex: 1000, - backgroundColor: "rgba(0, 0, 0, 0.8)", - display: "flex", - justifyContent: "center", - alignItems: "center" -}; - -const keyboardContainer = { - backgroundColor: "#1e1e1e", - padding: "16px", - borderRadius: "6px", - border: "1px solid #444", - zIndex: 1001, - maxWidth: "650px" -}; - -export default { - type: "Macro_KeyPress", - label: "Key Press (GUI-ONLY)", - description: ` -Press a single character or function key on a full keyboard overlay. -Shift/caps toggles uppercase/symbols. -F-keys are included, but pressing them won't store that value unless you remove them from "skip" logic, if desired. -`, - content: "Send Key Press to Foreground Window via Borealis Agent", - component: KeyPressNode -};