Agent Multi-Role Milestone 2

This commit is contained in:
2025-05-02 23:31:50 -06:00
parent 5dfcbcc403
commit 4c42f53cae
3 changed files with 276 additions and 238 deletions

View File

@ -36,7 +36,6 @@ class ConfigManager:
self.load() self.load()
def load(self): def load(self):
# load or initialize
if not os.path.exists(self.path): if not os.path.exists(self.path):
self.data = DEFAULT_CONFIG.copy() self.data = DEFAULT_CONFIG.copy()
self._write() self._write()
@ -44,12 +43,10 @@ class ConfigManager:
try: try:
with open(self.path, 'r') as f: with open(self.path, 'r') as f:
loaded = json.load(f) loaded = json.load(f)
# merge defaults
self.data = {**DEFAULT_CONFIG, **loaded} self.data = {**DEFAULT_CONFIG, **loaded}
except Exception as e: except Exception as e:
print(f"[WARN] Failed to parse config: {e}") print(f"[WARN] Failed to parse config: {e}")
self.data = DEFAULT_CONFIG.copy() self.data = DEFAULT_CONFIG.copy()
# track mtime
try: try:
self._last_mtime = os.path.getmtime(self.path) self._last_mtime = os.path.getmtime(self.path)
except Exception: except Exception:
@ -74,14 +71,12 @@ class ConfigManager:
return False return False
CONFIG = ConfigManager(CONFIG_PATH) CONFIG = ConfigManager(CONFIG_PATH)
# Purge saved regions on startup (fresh run)
CONFIG.data['regions'] = {} CONFIG.data['regions'] = {}
CONFIG._write() CONFIG._write()
# ////////////////////////////////////////////////////////////////////////// # //////////////////////////////////////////////////////////////////////////
# END CORE SECTION: CONFIG MANAGER # END CORE SECTION: CONFIG MANAGER
# ////////////////////////////////////////////////////////////////////////// # //////////////////////////////////////////////////////////////////////////
# Assign or persist agent_id
host = socket.gethostname().lower() host = socket.gethostname().lower()
stored_id = CONFIG.data.get('agent_id') stored_id = CONFIG.data.get('agent_id')
if stored_id: if stored_id:
@ -108,7 +103,6 @@ async def connect():
@sio.event @sio.event
async def disconnect(): async def disconnect():
print("[WebSocket] Disconnected from Borealis server.") print("[WebSocket] Disconnected from Borealis server.")
# reset tasks and overlays
for task in list(role_tasks.values()): for task in list(role_tasks.values()):
task.cancel() task.cancel()
role_tasks.clear() role_tasks.clear()
@ -116,50 +110,50 @@ async def disconnect():
try: widget.close() try: widget.close()
except: pass except: pass
overlay_widgets.clear() overlay_widgets.clear()
# purge regions on intentional disconnect
CONFIG.data['regions'].clear() CONFIG.data['regions'].clear()
CONFIG._write() CONFIG._write()
# reload settings
CONFIG.load() CONFIG.load()
@sio.on('agent_config') @sio.on('agent_config')
async def on_agent_config(cfg): async def on_agent_config(cfg):
print(f"[CONNECTED] Received config with {len(cfg.get('roles',[]))} roles.") print(f"[CONNECTED] Received config with {len(cfg.get('roles',[]))} roles.")
# determine removed roles
new_ids = {r.get('node_id') for r in cfg.get('roles', []) if r.get('node_id')} new_ids = {r.get('node_id') for r in cfg.get('roles', []) if r.get('node_id')}
old_ids = set(role_tasks.keys()) old_ids = set(role_tasks.keys())
removed = old_ids - new_ids removed = old_ids - new_ids
# Cancel removed roles
for rid in removed: for rid in removed:
# remove region config
if rid in CONFIG.data['regions']: if rid in CONFIG.data['regions']:
CONFIG.data['regions'].pop(rid, None) CONFIG.data['regions'].pop(rid, None)
# close overlay
w = overlay_widgets.pop(rid, None) w = overlay_widgets.pop(rid, None)
if w: if w:
try: w.close() try: w.close()
except: pass except: pass
if removed: if removed:
CONFIG._write() CONFIG._write()
# cancel existing and start new
# Cancel all existing to ensure clean state
for task in list(role_tasks.values()): for task in list(role_tasks.values()):
task.cancel() task.cancel()
role_tasks.clear() role_tasks.clear()
# Restart everything to ensure roles are re-applied
for role_cfg in cfg.get('roles', []): for role_cfg in cfg.get('roles', []):
nid = role_cfg.get('node_id')
if role_cfg.get('role') == 'screenshot': if role_cfg.get('role') == 'screenshot':
nid = role_cfg.get('node_id') task = asyncio.create_task(screenshot_task(role_cfg))
t = asyncio.create_task(screenshot_task(role_cfg)) role_tasks[nid] = task
role_tasks[nid] = t
# ////////////////////////////////////////////////////////////////////////// # //////////////////////////////////////////////////////////////////////////
# END CORE SECTION: WEBSOCKET SETUP & HANDLERS # END CORE SECTION: WEBSOCKET SETUP & HANDLERS
# ////////////////////////////////////////////////////////////////////////// # //////////////////////////////////////////////////////////////////////////
# ---------------- Overlay Widget ----------------
class ScreenshotRegion(QtWidgets.QWidget): class ScreenshotRegion(QtWidgets.QWidget):
def __init__(self, node_id, x=100, y=100, w=300, h=200): def __init__(self, node_id, x=100, y=100, w=300, h=200):
super().__init__() super().__init__()
self.node_id = node_id self.node_id = node_id
self.setGeometry(x, y, w, h) self.setGeometry(x, y, w, h)
self.setWindowFlags(QtCore.Qt.FramelessWindowHint|QtCore.Qt.WindowStaysOnTopHint) self.setWindowFlags(QtCore.Qt.FramelessWindowHint | QtCore.Qt.WindowStaysOnTopHint)
self.setAttribute(QtCore.Qt.WA_TranslucentBackground) self.setAttribute(QtCore.Qt.WA_TranslucentBackground)
self.drag_offset = None self.drag_offset = None
self.resizing = False self.resizing = False
@ -168,7 +162,7 @@ class ScreenshotRegion(QtWidgets.QWidget):
self.label = QtWidgets.QLabel(self) self.label = QtWidgets.QLabel(self)
self.label.setText(f"{node_id[:8]}") self.label.setText(f"{node_id[:8]}")
self.label.setStyleSheet("color: lime; background: transparent; font-size: 10px;") self.label.setStyleSheet("color: lime; background: transparent; font-size: 10px;")
self.label.move(8,4) self.label.move(8, 4)
self.setMouseTracking(True) self.setMouseTracking(True)
def paintEvent(self, event): def paintEvent(self, event):
@ -183,51 +177,72 @@ class ScreenshotRegion(QtWidgets.QWidget):
def mousePressEvent(self, e): def mousePressEvent(self, e):
if e.button()==QtCore.Qt.LeftButton: if e.button()==QtCore.Qt.LeftButton:
x,y = e.pos().x(), e.pos().y() x, y = e.pos().x(), e.pos().y()
if x>self.width()-self.resize_handle_size and y>self.height()-self.resize_handle_size: if x > self.width() - self.resize_handle_size and y > self.height() - self.resize_handle_size:
self.resizing=True self.resizing = True
else: else:
self.drag_offset = e.globalPos() - self.frameGeometry().topLeft() self.drag_offset = e.globalPos() - self.frameGeometry().topLeft()
def mouseMoveEvent(self, e): def mouseMoveEvent(self, e):
if self.resizing: if self.resizing:
nw = max(e.pos().x(),100) nw = max(e.pos().x(), 100)
nh = max(e.pos().y(),80) nh = max(e.pos().y(), 80)
self.resize(nw,nh) self.resize(nw, nh)
elif e.buttons()&QtCore.Qt.LeftButton and self.drag_offset: elif e.buttons() & QtCore.Qt.LeftButton and self.drag_offset:
self.move(e.globalPos()-self.drag_offset) self.move(e.globalPos() - self.drag_offset)
def mouseReleaseEvent(self,e): def mouseReleaseEvent(self, e):
self.resizing=False; self.drag_offset=None self.resizing = False
self.drag_offset = None
def get_geometry(self): def get_geometry(self):
g=self.geometry(); return (g.x(),g.y(),g.width(),g.height()) g = self.geometry()
return (g.x(), g.y(), g.width(), g.height())
# ---------------- Screenshot Task ---------------- # ---------------- Screenshot Task ----------------
async def screenshot_task(cfg): async def screenshot_task(cfg):
nid = cfg.get('node_id'); nid = cfg.get('node_id')
# initial region from config or payload # If existing region in config, honor that
r = CONFIG.data['regions'].get(nid) r = CONFIG.data['regions'].get(nid)
region = (r['x'],r['y'],r['w'],r['h']) if r else (cfg.get('x',100),cfg.get('y',100),cfg.get('w',300),cfg.get('h',200)) if r:
region = (r['x'], r['y'], r['w'], r['h'])
else:
region = (cfg.get('x', 100), cfg.get('y', 100), cfg.get('w', 300), cfg.get('h', 200))
CONFIG.data['regions'][nid] = {
'x': region[0], 'y': region[1], 'w': region[2], 'h': region[3]
}
CONFIG._write()
if nid not in overlay_widgets: if nid not in overlay_widgets:
widget = ScreenshotRegion(nid,*region) widget = ScreenshotRegion(nid, *region)
overlay_widgets[nid] = widget; widget.show() overlay_widgets[nid] = widget
interval = cfg.get('interval',1000)/1000.0 widget.show()
interval = cfg.get('interval', 1000) / 1000.0
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
executor = concurrent.futures.ThreadPoolExecutor(max_workers=CONFIG.data.get('max_task_workers',DEFAULT_CONFIG['max_task_workers'])) executor = concurrent.futures.ThreadPoolExecutor(
max_workers=CONFIG.data.get('max_task_workers', DEFAULT_CONFIG['max_task_workers'])
)
try: try:
while True: while True:
x,y,w,h = overlay_widgets[nid].get_geometry() x, y, w, h = overlay_widgets[nid].get_geometry()
# persist if changed
prev = CONFIG.data['regions'].get(nid) prev = CONFIG.data['regions'].get(nid)
if prev!={'x':x,'y':y,'w':w,'h':h}: new_geom = {'x': x, 'y': y, 'w': w, 'h': h}
CONFIG.data['regions'][nid]={'x':x,'y':y,'w':w,'h':h} if prev != new_geom:
CONFIG.data['regions'][nid] = new_geom
CONFIG._write() CONFIG._write()
grab = partial(ImageGrab.grab,bbox=(x,y,x+w,y+h)) grab = partial(ImageGrab.grab, bbox=(x, y, x+w, y+h))
img = await loop.run_in_executor(executor,grab) img = await loop.run_in_executor(executor, grab)
buf = BytesIO(); img.save(buf,format='PNG') buf = BytesIO()
img.save(buf, format='PNG')
encoded = base64.b64encode(buf.getvalue()).decode('utf-8') encoded = base64.b64encode(buf.getvalue()).decode('utf-8')
await sio.emit('agent_screenshot_task',{'agent_id':AGENT_ID,'node_id':nid,'image_base64':encoded}) await sio.emit('agent_screenshot_task', {
'agent_id': AGENT_ID,
'node_id': nid,
'image_base64': encoded,
'x': x, 'y': y, 'w': w, 'h': h # Bi-directional live-sync
})
await asyncio.sleep(interval) await asyncio.sleep(interval)
except asyncio.CancelledError: except asyncio.CancelledError:
return return
@ -238,24 +253,24 @@ async def screenshot_task(cfg):
async def config_watcher(): async def config_watcher():
while True: while True:
if CONFIG.watch(): pass if CONFIG.watch(): pass
await asyncio.sleep(CONFIG.data.get('config_file_watcher_interval',DEFAULT_CONFIG['config_file_watcher_interval'])) await asyncio.sleep(CONFIG.data.get('config_file_watcher_interval', DEFAULT_CONFIG['config_file_watcher_interval']))
# ////////////////////////////////////////////////////////////////////////// # //////////////////////////////////////////////////////////////////////////
# CORE SECTION: MAIN & EVENT LOOP (do not modify unless you know what youre doing) # CORE SECTION: MAIN & EVENT LOOP (do not modify unless you know what youre doing)
# ////////////////////////////////////////////////////////////////////////// # //////////////////////////////////////////////////////////////////////////
async def connect_loop(): async def connect_loop():
retry=5 retry = 5
while True: while True:
try: try:
url=CONFIG.data.get('borealis_server_url',DEFAULT_CONFIG['borealis_server_url']) url = CONFIG.data.get('borealis_server_url', DEFAULT_CONFIG['borealis_server_url'])
print(f"[WebSocket] Connecting to {url}...") print(f"[WebSocket] Connecting to {url}...")
await sio.connect(url,transports=['websocket']) await sio.connect(url, transports=['websocket'])
break break
except: except:
print(f"[WebSocket] Server unavailable, retrying in {retry}s...") print(f"[WebSocket] Server unavailable, retrying in {retry}s...")
await asyncio.sleep(retry) await asyncio.sleep(retry)
if __name__=='__main__': if __name__ == '__main__':
app = QtWidgets.QApplication(sys.argv) app = QtWidgets.QApplication(sys.argv)
loop = QEventLoop(app) loop = QEventLoop(app)
asyncio.set_event_loop(loop) asyncio.set_event_loop(loop)

View File

@ -1,6 +1,6 @@
////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/WebUI/src/nodes/Agent/Node_Agent.jsx ////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/WebUI/src/nodes/Agent/Node_Agent.jsx
import React, { useEffect, useState, useCallback, useMemo } from "react"; import React, { useEffect, useState, useCallback, useMemo, useRef } from "react";
import { Handle, Position, useReactFlow, useStore } from "reactflow"; import { Handle, Position, useReactFlow, useStore } from "reactflow";
const BorealisAgentNode = ({ id, data }) => { const BorealisAgentNode = ({ id, data }) => {
@ -9,18 +9,20 @@ const BorealisAgentNode = ({ id, data }) => {
const [agents, setAgents] = useState({}); const [agents, setAgents] = useState({});
const [selectedAgent, setSelectedAgent] = useState(data.agent_id || ""); const [selectedAgent, setSelectedAgent] = useState(data.agent_id || "");
const [isConnected, setIsConnected] = useState(false); const [isConnected, setIsConnected] = useState(false);
const prevRolesRef = useRef([]);
// Build a normalized list [{id, status}, ...]
const agentList = useMemo(() => { const agentList = useMemo(() => {
if (Array.isArray(agents)) { if (!agents || typeof agents !== "object") return [];
return agents.map((a) => ({ id: a.id, status: a.status })); return Object.entries(agents)
} else if (agents && typeof agents === "object") { .map(([aid, info]) => ({
return Object.entries(agents).map(([aid, info]) => ({ id: aid, status: info.status })); id: aid,
} status: info?.status || "offline",
return []; last_seen: info?.last_seen || 0
}))
.filter(({ status }) => status !== "offline")
.sort((a, b) => b.last_seen - a.last_seen);
}, [agents]); }, [agents]);
// Fetch agents from backend
useEffect(() => { useEffect(() => {
const fetchAgents = () => { const fetchAgents = () => {
fetch("/api/agents") fetch("/api/agents")
@ -29,11 +31,10 @@ const BorealisAgentNode = ({ id, data }) => {
.catch(() => {}); .catch(() => {});
}; };
fetchAgents(); fetchAgents();
const interval = setInterval(fetchAgents, 5000); const interval = setInterval(fetchAgents, 4000);
return () => clearInterval(interval); return () => clearInterval(interval);
}, []); }, []);
// Persist selectedAgent and reset connection on change
useEffect(() => { useEffect(() => {
setNodes((nds) => setNodes((nds) =>
nds.map((n) => nds.map((n) =>
@ -43,7 +44,6 @@ const BorealisAgentNode = ({ id, data }) => {
setIsConnected(false); setIsConnected(false);
}, [selectedAgent]); }, [selectedAgent]);
// Compute attached role node IDs for this agent node
const attachedRoleIds = useMemo( const attachedRoleIds = useMemo(
() => () =>
edges edges
@ -52,7 +52,6 @@ const BorealisAgentNode = ({ id, data }) => {
[edges, id] [edges, id]
); );
// Build role payloads using the instruction registry
const getAttachedRoles = useCallback(() => { const getAttachedRoles = useCallback(() => {
const allNodes = getNodes(); const allNodes = getNodes();
return attachedRoleIds return attachedRoleIds
@ -63,42 +62,54 @@ const BorealisAgentNode = ({ id, data }) => {
.filter((r) => r); .filter((r) => r);
}, [attachedRoleIds, getNodes]); }, [attachedRoleIds, getNodes]);
// Connect: send roles to server const provisionRoles = useCallback((roles) => {
const handleConnect = useCallback(() => { if (!selectedAgent || roles.length === 0) return;
if (!selectedAgent) return;
const roles = getAttachedRoles();
fetch("/api/agent/provision", { fetch("/api/agent/provision", {
method: "POST", method: "POST",
headers: { "Content-Type": "application/json" }, headers: { "Content-Type": "application/json" },
body: JSON.stringify({ agent_id: selectedAgent, roles }), body: JSON.stringify({ agent_id: selectedAgent, roles })
}) })
.then(() => setIsConnected(true)) .then(() => {
setIsConnected(true);
prevRolesRef.current = roles;
})
.catch(() => {}); .catch(() => {});
}, [selectedAgent, getAttachedRoles]); }, [selectedAgent]);
const handleConnect = useCallback(() => {
const roles = getAttachedRoles();
provisionRoles(roles);
}, [getAttachedRoles, provisionRoles]);
// Disconnect: clear roles on server
const handleDisconnect = useCallback(() => { const handleDisconnect = useCallback(() => {
if (!selectedAgent) return; if (!selectedAgent) return;
fetch("/api/agent/provision", { fetch("/api/agent/provision", {
method: "POST", method: "POST",
headers: { "Content-Type": "application/json" }, headers: { "Content-Type": "application/json" },
body: JSON.stringify({ agent_id: selectedAgent, roles: [] }), body: JSON.stringify({ agent_id: selectedAgent, roles: [] })
}) })
.then(() => setIsConnected(false)) .then(() => {
setIsConnected(false);
prevRolesRef.current = [];
})
.catch(() => {}); .catch(() => {});
}, [selectedAgent]); }, [selectedAgent]);
// Hot-update roles when attachedRoleIds change
useEffect(() => { useEffect(() => {
if (isConnected) { const newRoles = getAttachedRoles();
const roles = getAttachedRoles(); const prevSerialized = JSON.stringify(prevRolesRef.current || []);
fetch("/api/agent/provision", { const newSerialized = JSON.stringify(newRoles);
method: "POST", if (isConnected && newSerialized !== prevSerialized) {
headers: { "Content-Type": "application/json" }, provisionRoles(newRoles);
body: JSON.stringify({ agent_id: selectedAgent, roles }),
}).catch(() => {});
} }
}, [attachedRoleIds]); }, [attachedRoleIds, isConnected]);
const selectedAgentStatus = useMemo(() => {
if (!selectedAgent) return "Unassigned";
const agent = agents[selectedAgent];
if (!agent) return "Reconnecting...";
return agent.status === "provisioned" ? "Connected" : "Available";
}, [agents, selectedAgent]);
return ( return (
<div className="borealis-node"> <div className="borealis-node">
@ -119,14 +130,11 @@ const BorealisAgentNode = ({ id, data }) => {
style={{ width: "100%", marginBottom: "6px", fontSize: "9px" }} style={{ width: "100%", marginBottom: "6px", fontSize: "9px" }}
> >
<option value="">-- Select --</option> <option value="">-- Select --</option>
{agentList.map(({ id: aid, status }) => { {agentList.map(({ id: aid, status }) => (
const labelText = status === "provisioned" ? "(Connected)" : "(Ready to Connect)"; <option key={aid} value={aid}>
return ( {aid} ({status})
<option key={aid} value={aid}> </option>
{aid} {labelText} ))}
</option>
);
})}
</select> </select>
{isConnected ? ( {isConnected ? (
@ -149,7 +157,9 @@ const BorealisAgentNode = ({ id, data }) => {
<hr style={{ margin: "6px 0", borderColor: "#444" }} /> <hr style={{ margin: "6px 0", borderColor: "#444" }} />
<div style={{ fontSize: "8px", color: "#aaa" }}> <div style={{ fontSize: "8px", color: "#aaa" }}>
Attach <strong>Agent Role Nodes</strong> to define roles for this agent. Roles will be provisioned automatically. Status: <strong>{selectedAgentStatus}</strong>
<br />
Attach <strong>Agent Role Nodes</strong> to define live behavior.
</div> </div>
</div> </div>
</div> </div>

View File

@ -6,181 +6,194 @@ import ShareIcon from "@mui/icons-material/Share";
import IconButton from "@mui/material/IconButton"; import IconButton from "@mui/material/IconButton";
if (!window.BorealisValueBus) { if (!window.BorealisValueBus) {
window.BorealisValueBus = {}; window.BorealisValueBus = {};
} }
if (!window.BorealisUpdateRate) { if (!window.BorealisUpdateRate) {
window.BorealisUpdateRate = 100; window.BorealisUpdateRate = 100;
} }
const ScreenshotInstructionNode = ({ id, data }) => { const ScreenshotInstructionNode = ({ id, data }) => {
const { setNodes, getNodes } = useReactFlow(); const { setNodes, getNodes } = useReactFlow();
const edges = useStore(state => state.edges); const edges = useStore(state => state.edges);
const [interval, setInterval] = useState(data?.interval || 1000); const [interval, setInterval] = useState(data?.interval || 1000);
const [region, setRegion] = useState({ const [region, setRegion] = useState({
x: data?.x ?? 250, x: data?.x ?? 250,
y: data?.y ?? 100, y: data?.y ?? 100,
w: data?.w ?? 300, w: data?.w ?? 300,
h: data?.h ?? 200, h: data?.h ?? 200,
}); });
const [visible, setVisible] = useState(data?.visible ?? true); const [visible, setVisible] = useState(data?.visible ?? true);
const [alias, setAlias] = useState(data?.alias || ""); const [alias, setAlias] = useState(data?.alias || "");
const [imageBase64, setImageBase64] = useState(""); const [imageBase64, setImageBase64] = useState("");
const base64Ref = useRef(""); const base64Ref = useRef("");
const regionRef = useRef(region);
const handleCopyLiveViewLink = () => { // Push current state into BorealisValueBus at intervals
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner"); useEffect(() => {
if (!agentEdge) { const intervalId = setInterval(() => {
alert("No upstream agent connection found."); const val = base64Ref.current;
return; if (val) {
window.BorealisValueBus[id] = val;
setNodes(nds =>
nds.map(n =>
n.id === id ? { ...n, data: { ...n.data, value: val } } : n
)
);
}
}, window.BorealisUpdateRate || 100);
return () => clearInterval(intervalId);
}, [id, setNodes]);
// Listen for agent screenshot + overlay updates
useEffect(() => {
const socket = window.BorealisSocket;
if (!socket) return;
const handleScreenshot = (payload) => {
if (payload?.node_id !== id || !payload.image_base64) return;
base64Ref.current = payload.image_base64;
setImageBase64(payload.image_base64);
window.BorealisValueBus[id] = payload.image_base64;
// If geometry changed from agent side, sync into UI
const { x, y, w, h } = payload;
if (x !== undefined && y !== undefined && w !== undefined && h !== undefined) {
const newRegion = { x, y, w, h };
const prev = regionRef.current;
const changed = Object.entries(newRegion).some(([k, v]) => prev[k] !== v);
if (changed) {
regionRef.current = newRegion;
setRegion(newRegion);
setNodes(nds =>
nds.map(n =>
n.id === id ? { ...n, data: { ...n.data, ...newRegion } } : n
)
);
} }
}
const agentNode = getNodes().find(n => n.id === agentEdge.source);
const selectedAgentId = agentNode?.data?.agent_id;
if (!selectedAgentId) {
alert("Upstream agent node does not have a selected agent.");
return;
}
const liveUrl = `${window.location.origin}/api/agent/${selectedAgentId}/node/${id}/screenshot/live`;
navigator.clipboard.writeText(liveUrl)
.then(() => console.log(`[Clipboard] Copied Live View URL: ${liveUrl}`))
.catch(err => console.error("Clipboard copy failed:", err));
}; };
useEffect(() => { socket.on("agent_screenshot_task", handleScreenshot);
const intervalId = setInterval(() => { return () => socket.off("agent_screenshot_task", handleScreenshot);
const val = base64Ref.current; }, [id, setNodes]);
console.log(`[Screenshot Node] setInterval update. Current base64 length: ${val?.length || 0}`); // Bi-directional instruction export
window.__BorealisInstructionNodes = window.__BorealisInstructionNodes || {};
window.__BorealisInstructionNodes[id] = () => ({
node_id: id,
role: "screenshot",
interval,
visible,
alias,
...regionRef.current
});
if (!val) return; // Manual live view copy
const handleCopyLiveViewLink = () => {
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner");
const agentNode = getNodes().find(n => n.id === agentEdge?.source);
const selectedAgentId = agentNode?.data?.agent_id;
window.BorealisValueBus[id] = val; if (!selectedAgentId) {
setNodes(nds => alert("No valid agent connection found.");
nds.map(n => return;
n.id === id }
? { ...n, data: { ...n.data, value: val } }
: n
)
);
}, window.BorealisUpdateRate || 100);
return () => clearInterval(intervalId); const liveUrl = `${window.location.origin}/api/agent/${selectedAgentId}/node/${id}/screenshot/live`;
}, [id, setNodes]); navigator.clipboard.writeText(liveUrl)
.then(() => console.log(`[Clipboard] Live View URL copied: ${liveUrl}`))
.catch(err => console.error("Clipboard copy failed:", err));
};
useEffect(() => { return (
const socket = window.BorealisSocket || null; <div className="borealis-node" style={{ position: "relative" }}>
if (!socket) { <Handle type="target" position={Position.Left} className="borealis-handle" />
console.warn("[Screenshot Node] BorealisSocket not available"); <Handle type="source" position={Position.Right} className="borealis-handle" />
return;
}
console.log(`[Screenshot Node] Listening for agent_screenshot_task with node_id: ${id}`); <div className="borealis-node-header">Agent Role: Screenshot</div>
<div className="borealis-node-content" style={{ fontSize: "9px" }}>
<label>Update Interval (ms):</label>
<input
type="number"
min="100"
step="100"
value={interval}
onChange={(e) => setInterval(Number(e.target.value))}
style={{ width: "100%", marginBottom: "4px" }}
/>
const handleScreenshot = (payload) => { <label>Region X / Y / W / H:</label>
console.log("[Screenshot Node] Received payload:", payload); <div style={{ display: "flex", gap: "4px", marginBottom: "4px" }}>
<input type="number" value={region.x} onChange={(e) => {
if (payload?.node_id === id && payload?.image_base64) { const x = Number(e.target.value);
base64Ref.current = payload.image_base64; const updated = { ...region, x }; setRegion(updated); regionRef.current = updated;
setImageBase64(payload.image_base64); }} style={{ width: "25%" }} />
window.BorealisValueBus[id] = payload.image_base64; <input type="number" value={region.y} onChange={(e) => {
const y = Number(e.target.value);
console.log(`[Screenshot Node] Updated base64Ref and ValueBus for ${id}, length: ${payload.image_base64.length}`); const updated = { ...region, y }; setRegion(updated); regionRef.current = updated;
} else { }} style={{ width: "25%" }} />
console.log(`[Screenshot Node] Ignored payload for mismatched node_id (${payload?.node_id})`); <input type="number" value={region.w} onChange={(e) => {
} const w = Number(e.target.value);
}; const updated = { ...region, w }; setRegion(updated); regionRef.current = updated;
}} style={{ width: "25%" }} />
socket.on("agent_screenshot_task", handleScreenshot); <input type="number" value={region.h} onChange={(e) => {
return () => socket.off("agent_screenshot_task", handleScreenshot); const h = Number(e.target.value);
}, [id]); const updated = { ...region, h }; setRegion(updated); regionRef.current = updated;
}} style={{ width: "25%" }} />
window.__BorealisInstructionNodes = window.__BorealisInstructionNodes || {};
window.__BorealisInstructionNodes[id] = () => ({
node_id: id,
role: "screenshot",
interval,
visible,
alias,
...region
});
return (
<div className="borealis-node" style={{ position: "relative" }}>
<Handle type="target" position={Position.Left} className="borealis-handle" />
<Handle type="source" position={Position.Right} className="borealis-handle" />
<div className="borealis-node-header">Agent Role: Screenshot</div>
<div className="borealis-node-content" style={{ fontSize: "9px" }}>
<label>Update Interval (ms):</label>
<input
type="number"
min="100"
step="100"
value={interval}
onChange={(e) => setInterval(Number(e.target.value))}
style={{ width: "100%", marginBottom: "4px" }}
/>
<label>Region X / Y / W / H:</label>
<div style={{ display: "flex", gap: "4px", marginBottom: "4px" }}>
<input type="number" value={region.x} onChange={(e) => setRegion({ ...region, x: Number(e.target.value) })} style={{ width: "25%" }} />
<input type="number" value={region.y} onChange={(e) => setRegion({ ...region, y: Number(e.target.value) })} style={{ width: "25%" }} />
<input type="number" value={region.w} onChange={(e) => setRegion({ ...region, w: Number(e.target.value) })} style={{ width: "25%" }} />
<input type="number" value={region.h} onChange={(e) => setRegion({ ...region, h: Number(e.target.value) })} style={{ width: "25%" }} />
</div>
<div style={{ marginBottom: "4px" }}>
<label>
<input
type="checkbox"
checked={visible}
onChange={() => setVisible(!visible)}
style={{ marginRight: "4px" }}
/>
Show Overlay on Agent
</label>
</div>
<label>Overlay Label:</label>
<input
type="text"
value={alias}
onChange={(e) => setAlias(e.target.value)}
placeholder="Label (optional)"
style={{ width: "100%", fontSize: "9px", marginBottom: "6px" }}
/>
<div style={{ textAlign: "center", fontSize: "8px", color: "#aaa" }}>
{imageBase64
? `Last image: ${Math.round(imageBase64.length / 1024)} KB`
: "Awaiting Screenshot Data..."}
</div>
</div>
<div style={{ position: "absolute", top: 4, right: 4 }}>
<IconButton size="small" onClick={handleCopyLiveViewLink}>
<ShareIcon style={{ fontSize: 14 }} />
</IconButton>
</div>
</div> </div>
);
<div style={{ marginBottom: "4px" }}>
<label>
<input
type="checkbox"
checked={visible}
onChange={() => setVisible(!visible)}
style={{ marginRight: "4px" }}
/>
Show Overlay on Agent
</label>
</div>
<label>Overlay Label:</label>
<input
type="text"
value={alias}
onChange={(e) => setAlias(e.target.value)}
placeholder="Label (optional)"
style={{ width: "100%", fontSize: "9px", marginBottom: "6px" }}
/>
<div style={{ textAlign: "center", fontSize: "8px", color: "#aaa" }}>
{imageBase64
? `Last image: ${Math.round(imageBase64.length / 1024)} KB`
: "Awaiting Screenshot Data..."}
</div>
</div>
<div style={{ position: "absolute", top: 4, right: 4 }}>
<IconButton size="small" onClick={handleCopyLiveViewLink}>
<ShareIcon style={{ fontSize: 14 }} />
</IconButton>
</div>
</div>
);
}; };
export default { export default {
type: "Agent_Role_Screenshot", type: "Agent_Role_Screenshot",
label: "Agent Role: Screenshot", label: "Agent Role: Screenshot",
description: ` description: `
Agent Role Node: Screenshot Region Agent Role Node: Screenshot Region
- Defines a single region capture role - Defines a single region capture role
- Allows custom update interval and overlay - Allows custom update interval and overlay
- Emits captured base64 PNG data from agent - Emits captured base64 PNG data from agent
`.trim(), `.trim(),
content: "Capture screenshot region via agent", content: "Capture screenshot region via agent",
component: ScreenshotInstructionNode component: ScreenshotInstructionNode
}; };