Agent Multi-Role Milestone 2
This commit is contained in:
@ -1,6 +1,6 @@
|
||||
////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/WebUI/src/nodes/Agent/Node_Agent.jsx
|
||||
|
||||
import React, { useEffect, useState, useCallback, useMemo } from "react";
|
||||
import React, { useEffect, useState, useCallback, useMemo, useRef } from "react";
|
||||
import { Handle, Position, useReactFlow, useStore } from "reactflow";
|
||||
|
||||
const BorealisAgentNode = ({ id, data }) => {
|
||||
@ -9,18 +9,20 @@ const BorealisAgentNode = ({ id, data }) => {
|
||||
const [agents, setAgents] = useState({});
|
||||
const [selectedAgent, setSelectedAgent] = useState(data.agent_id || "");
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
const prevRolesRef = useRef([]);
|
||||
|
||||
// Build a normalized list [{id, status}, ...]
|
||||
const agentList = useMemo(() => {
|
||||
if (Array.isArray(agents)) {
|
||||
return agents.map((a) => ({ id: a.id, status: a.status }));
|
||||
} else if (agents && typeof agents === "object") {
|
||||
return Object.entries(agents).map(([aid, info]) => ({ id: aid, status: info.status }));
|
||||
}
|
||||
return [];
|
||||
if (!agents || typeof agents !== "object") return [];
|
||||
return Object.entries(agents)
|
||||
.map(([aid, info]) => ({
|
||||
id: aid,
|
||||
status: info?.status || "offline",
|
||||
last_seen: info?.last_seen || 0
|
||||
}))
|
||||
.filter(({ status }) => status !== "offline")
|
||||
.sort((a, b) => b.last_seen - a.last_seen);
|
||||
}, [agents]);
|
||||
|
||||
// Fetch agents from backend
|
||||
useEffect(() => {
|
||||
const fetchAgents = () => {
|
||||
fetch("/api/agents")
|
||||
@ -29,11 +31,10 @@ const BorealisAgentNode = ({ id, data }) => {
|
||||
.catch(() => {});
|
||||
};
|
||||
fetchAgents();
|
||||
const interval = setInterval(fetchAgents, 5000);
|
||||
const interval = setInterval(fetchAgents, 4000);
|
||||
return () => clearInterval(interval);
|
||||
}, []);
|
||||
|
||||
// Persist selectedAgent and reset connection on change
|
||||
useEffect(() => {
|
||||
setNodes((nds) =>
|
||||
nds.map((n) =>
|
||||
@ -43,7 +44,6 @@ const BorealisAgentNode = ({ id, data }) => {
|
||||
setIsConnected(false);
|
||||
}, [selectedAgent]);
|
||||
|
||||
// Compute attached role node IDs for this agent node
|
||||
const attachedRoleIds = useMemo(
|
||||
() =>
|
||||
edges
|
||||
@ -52,7 +52,6 @@ const BorealisAgentNode = ({ id, data }) => {
|
||||
[edges, id]
|
||||
);
|
||||
|
||||
// Build role payloads using the instruction registry
|
||||
const getAttachedRoles = useCallback(() => {
|
||||
const allNodes = getNodes();
|
||||
return attachedRoleIds
|
||||
@ -63,42 +62,54 @@ const BorealisAgentNode = ({ id, data }) => {
|
||||
.filter((r) => r);
|
||||
}, [attachedRoleIds, getNodes]);
|
||||
|
||||
// Connect: send roles to server
|
||||
const handleConnect = useCallback(() => {
|
||||
if (!selectedAgent) return;
|
||||
const roles = getAttachedRoles();
|
||||
const provisionRoles = useCallback((roles) => {
|
||||
if (!selectedAgent || roles.length === 0) return;
|
||||
fetch("/api/agent/provision", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ agent_id: selectedAgent, roles }),
|
||||
body: JSON.stringify({ agent_id: selectedAgent, roles })
|
||||
})
|
||||
.then(() => setIsConnected(true))
|
||||
.then(() => {
|
||||
setIsConnected(true);
|
||||
prevRolesRef.current = roles;
|
||||
})
|
||||
.catch(() => {});
|
||||
}, [selectedAgent, getAttachedRoles]);
|
||||
}, [selectedAgent]);
|
||||
|
||||
const handleConnect = useCallback(() => {
|
||||
const roles = getAttachedRoles();
|
||||
provisionRoles(roles);
|
||||
}, [getAttachedRoles, provisionRoles]);
|
||||
|
||||
// Disconnect: clear roles on server
|
||||
const handleDisconnect = useCallback(() => {
|
||||
if (!selectedAgent) return;
|
||||
fetch("/api/agent/provision", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ agent_id: selectedAgent, roles: [] }),
|
||||
body: JSON.stringify({ agent_id: selectedAgent, roles: [] })
|
||||
})
|
||||
.then(() => setIsConnected(false))
|
||||
.then(() => {
|
||||
setIsConnected(false);
|
||||
prevRolesRef.current = [];
|
||||
})
|
||||
.catch(() => {});
|
||||
}, [selectedAgent]);
|
||||
|
||||
// Hot-update roles when attachedRoleIds change
|
||||
useEffect(() => {
|
||||
if (isConnected) {
|
||||
const roles = getAttachedRoles();
|
||||
fetch("/api/agent/provision", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ agent_id: selectedAgent, roles }),
|
||||
}).catch(() => {});
|
||||
const newRoles = getAttachedRoles();
|
||||
const prevSerialized = JSON.stringify(prevRolesRef.current || []);
|
||||
const newSerialized = JSON.stringify(newRoles);
|
||||
if (isConnected && newSerialized !== prevSerialized) {
|
||||
provisionRoles(newRoles);
|
||||
}
|
||||
}, [attachedRoleIds]);
|
||||
}, [attachedRoleIds, isConnected]);
|
||||
|
||||
const selectedAgentStatus = useMemo(() => {
|
||||
if (!selectedAgent) return "Unassigned";
|
||||
const agent = agents[selectedAgent];
|
||||
if (!agent) return "Reconnecting...";
|
||||
return agent.status === "provisioned" ? "Connected" : "Available";
|
||||
}, [agents, selectedAgent]);
|
||||
|
||||
return (
|
||||
<div className="borealis-node">
|
||||
@ -119,14 +130,11 @@ const BorealisAgentNode = ({ id, data }) => {
|
||||
style={{ width: "100%", marginBottom: "6px", fontSize: "9px" }}
|
||||
>
|
||||
<option value="">-- Select --</option>
|
||||
{agentList.map(({ id: aid, status }) => {
|
||||
const labelText = status === "provisioned" ? "(Connected)" : "(Ready to Connect)";
|
||||
return (
|
||||
<option key={aid} value={aid}>
|
||||
{aid} {labelText}
|
||||
</option>
|
||||
);
|
||||
})}
|
||||
{agentList.map(({ id: aid, status }) => (
|
||||
<option key={aid} value={aid}>
|
||||
{aid} ({status})
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
|
||||
{isConnected ? (
|
||||
@ -149,7 +157,9 @@ const BorealisAgentNode = ({ id, data }) => {
|
||||
<hr style={{ margin: "6px 0", borderColor: "#444" }} />
|
||||
|
||||
<div style={{ fontSize: "8px", color: "#aaa" }}>
|
||||
Attach <strong>Agent Role Nodes</strong> to define roles for this agent. Roles will be provisioned automatically.
|
||||
Status: <strong>{selectedAgentStatus}</strong>
|
||||
<br />
|
||||
Attach <strong>Agent Role Nodes</strong> to define live behavior.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -6,181 +6,194 @@ import ShareIcon from "@mui/icons-material/Share";
|
||||
import IconButton from "@mui/material/IconButton";
|
||||
|
||||
if (!window.BorealisValueBus) {
|
||||
window.BorealisValueBus = {};
|
||||
window.BorealisValueBus = {};
|
||||
}
|
||||
|
||||
if (!window.BorealisUpdateRate) {
|
||||
window.BorealisUpdateRate = 100;
|
||||
window.BorealisUpdateRate = 100;
|
||||
}
|
||||
|
||||
const ScreenshotInstructionNode = ({ id, data }) => {
|
||||
const { setNodes, getNodes } = useReactFlow();
|
||||
const edges = useStore(state => state.edges);
|
||||
const { setNodes, getNodes } = useReactFlow();
|
||||
const edges = useStore(state => state.edges);
|
||||
|
||||
const [interval, setInterval] = useState(data?.interval || 1000);
|
||||
const [region, setRegion] = useState({
|
||||
x: data?.x ?? 250,
|
||||
y: data?.y ?? 100,
|
||||
w: data?.w ?? 300,
|
||||
h: data?.h ?? 200,
|
||||
});
|
||||
const [visible, setVisible] = useState(data?.visible ?? true);
|
||||
const [alias, setAlias] = useState(data?.alias || "");
|
||||
const [imageBase64, setImageBase64] = useState("");
|
||||
const [interval, setInterval] = useState(data?.interval || 1000);
|
||||
const [region, setRegion] = useState({
|
||||
x: data?.x ?? 250,
|
||||
y: data?.y ?? 100,
|
||||
w: data?.w ?? 300,
|
||||
h: data?.h ?? 200,
|
||||
});
|
||||
const [visible, setVisible] = useState(data?.visible ?? true);
|
||||
const [alias, setAlias] = useState(data?.alias || "");
|
||||
const [imageBase64, setImageBase64] = useState("");
|
||||
|
||||
const base64Ref = useRef("");
|
||||
const base64Ref = useRef("");
|
||||
const regionRef = useRef(region);
|
||||
|
||||
const handleCopyLiveViewLink = () => {
|
||||
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner");
|
||||
if (!agentEdge) {
|
||||
alert("No upstream agent connection found.");
|
||||
return;
|
||||
// Push current state into BorealisValueBus at intervals
|
||||
useEffect(() => {
|
||||
const intervalId = setInterval(() => {
|
||||
const val = base64Ref.current;
|
||||
if (val) {
|
||||
window.BorealisValueBus[id] = val;
|
||||
setNodes(nds =>
|
||||
nds.map(n =>
|
||||
n.id === id ? { ...n, data: { ...n.data, value: val } } : n
|
||||
)
|
||||
);
|
||||
}
|
||||
}, window.BorealisUpdateRate || 100);
|
||||
return () => clearInterval(intervalId);
|
||||
}, [id, setNodes]);
|
||||
|
||||
// Listen for agent screenshot + overlay updates
|
||||
useEffect(() => {
|
||||
const socket = window.BorealisSocket;
|
||||
if (!socket) return;
|
||||
|
||||
const handleScreenshot = (payload) => {
|
||||
if (payload?.node_id !== id || !payload.image_base64) return;
|
||||
|
||||
base64Ref.current = payload.image_base64;
|
||||
setImageBase64(payload.image_base64);
|
||||
window.BorealisValueBus[id] = payload.image_base64;
|
||||
|
||||
// If geometry changed from agent side, sync into UI
|
||||
const { x, y, w, h } = payload;
|
||||
if (x !== undefined && y !== undefined && w !== undefined && h !== undefined) {
|
||||
const newRegion = { x, y, w, h };
|
||||
const prev = regionRef.current;
|
||||
const changed = Object.entries(newRegion).some(([k, v]) => prev[k] !== v);
|
||||
|
||||
if (changed) {
|
||||
regionRef.current = newRegion;
|
||||
setRegion(newRegion);
|
||||
setNodes(nds =>
|
||||
nds.map(n =>
|
||||
n.id === id ? { ...n, data: { ...n.data, ...newRegion } } : n
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const agentNode = getNodes().find(n => n.id === agentEdge.source);
|
||||
const selectedAgentId = agentNode?.data?.agent_id;
|
||||
|
||||
if (!selectedAgentId) {
|
||||
alert("Upstream agent node does not have a selected agent.");
|
||||
return;
|
||||
}
|
||||
|
||||
const liveUrl = `${window.location.origin}/api/agent/${selectedAgentId}/node/${id}/screenshot/live`;
|
||||
navigator.clipboard.writeText(liveUrl)
|
||||
.then(() => console.log(`[Clipboard] Copied Live View URL: ${liveUrl}`))
|
||||
.catch(err => console.error("Clipboard copy failed:", err));
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const intervalId = setInterval(() => {
|
||||
const val = base64Ref.current;
|
||||
socket.on("agent_screenshot_task", handleScreenshot);
|
||||
return () => socket.off("agent_screenshot_task", handleScreenshot);
|
||||
}, [id, setNodes]);
|
||||
|
||||
console.log(`[Screenshot Node] setInterval update. Current base64 length: ${val?.length || 0}`);
|
||||
// Bi-directional instruction export
|
||||
window.__BorealisInstructionNodes = window.__BorealisInstructionNodes || {};
|
||||
window.__BorealisInstructionNodes[id] = () => ({
|
||||
node_id: id,
|
||||
role: "screenshot",
|
||||
interval,
|
||||
visible,
|
||||
alias,
|
||||
...regionRef.current
|
||||
});
|
||||
|
||||
if (!val) return;
|
||||
// Manual live view copy
|
||||
const handleCopyLiveViewLink = () => {
|
||||
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner");
|
||||
const agentNode = getNodes().find(n => n.id === agentEdge?.source);
|
||||
const selectedAgentId = agentNode?.data?.agent_id;
|
||||
|
||||
window.BorealisValueBus[id] = val;
|
||||
setNodes(nds =>
|
||||
nds.map(n =>
|
||||
n.id === id
|
||||
? { ...n, data: { ...n.data, value: val } }
|
||||
: n
|
||||
)
|
||||
);
|
||||
}, window.BorealisUpdateRate || 100);
|
||||
if (!selectedAgentId) {
|
||||
alert("No valid agent connection found.");
|
||||
return;
|
||||
}
|
||||
|
||||
return () => clearInterval(intervalId);
|
||||
}, [id, setNodes]);
|
||||
const liveUrl = `${window.location.origin}/api/agent/${selectedAgentId}/node/${id}/screenshot/live`;
|
||||
navigator.clipboard.writeText(liveUrl)
|
||||
.then(() => console.log(`[Clipboard] Live View URL copied: ${liveUrl}`))
|
||||
.catch(err => console.error("Clipboard copy failed:", err));
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const socket = window.BorealisSocket || null;
|
||||
if (!socket) {
|
||||
console.warn("[Screenshot Node] BorealisSocket not available");
|
||||
return;
|
||||
}
|
||||
return (
|
||||
<div className="borealis-node" style={{ position: "relative" }}>
|
||||
<Handle type="target" position={Position.Left} className="borealis-handle" />
|
||||
<Handle type="source" position={Position.Right} className="borealis-handle" />
|
||||
|
||||
console.log(`[Screenshot Node] Listening for agent_screenshot_task with node_id: ${id}`);
|
||||
<div className="borealis-node-header">Agent Role: Screenshot</div>
|
||||
<div className="borealis-node-content" style={{ fontSize: "9px" }}>
|
||||
<label>Update Interval (ms):</label>
|
||||
<input
|
||||
type="number"
|
||||
min="100"
|
||||
step="100"
|
||||
value={interval}
|
||||
onChange={(e) => setInterval(Number(e.target.value))}
|
||||
style={{ width: "100%", marginBottom: "4px" }}
|
||||
/>
|
||||
|
||||
const handleScreenshot = (payload) => {
|
||||
console.log("[Screenshot Node] Received payload:", payload);
|
||||
|
||||
if (payload?.node_id === id && payload?.image_base64) {
|
||||
base64Ref.current = payload.image_base64;
|
||||
setImageBase64(payload.image_base64);
|
||||
window.BorealisValueBus[id] = payload.image_base64;
|
||||
|
||||
console.log(`[Screenshot Node] Updated base64Ref and ValueBus for ${id}, length: ${payload.image_base64.length}`);
|
||||
} else {
|
||||
console.log(`[Screenshot Node] Ignored payload for mismatched node_id (${payload?.node_id})`);
|
||||
}
|
||||
};
|
||||
|
||||
socket.on("agent_screenshot_task", handleScreenshot);
|
||||
return () => socket.off("agent_screenshot_task", handleScreenshot);
|
||||
}, [id]);
|
||||
|
||||
window.__BorealisInstructionNodes = window.__BorealisInstructionNodes || {};
|
||||
window.__BorealisInstructionNodes[id] = () => ({
|
||||
node_id: id,
|
||||
role: "screenshot",
|
||||
interval,
|
||||
visible,
|
||||
alias,
|
||||
...region
|
||||
});
|
||||
|
||||
return (
|
||||
<div className="borealis-node" style={{ position: "relative" }}>
|
||||
<Handle type="target" position={Position.Left} className="borealis-handle" />
|
||||
<Handle type="source" position={Position.Right} className="borealis-handle" />
|
||||
|
||||
<div className="borealis-node-header">Agent Role: Screenshot</div>
|
||||
<div className="borealis-node-content" style={{ fontSize: "9px" }}>
|
||||
<label>Update Interval (ms):</label>
|
||||
<input
|
||||
type="number"
|
||||
min="100"
|
||||
step="100"
|
||||
value={interval}
|
||||
onChange={(e) => setInterval(Number(e.target.value))}
|
||||
style={{ width: "100%", marginBottom: "4px" }}
|
||||
/>
|
||||
|
||||
<label>Region X / Y / W / H:</label>
|
||||
<div style={{ display: "flex", gap: "4px", marginBottom: "4px" }}>
|
||||
<input type="number" value={region.x} onChange={(e) => setRegion({ ...region, x: Number(e.target.value) })} style={{ width: "25%" }} />
|
||||
<input type="number" value={region.y} onChange={(e) => setRegion({ ...region, y: Number(e.target.value) })} style={{ width: "25%" }} />
|
||||
<input type="number" value={region.w} onChange={(e) => setRegion({ ...region, w: Number(e.target.value) })} style={{ width: "25%" }} />
|
||||
<input type="number" value={region.h} onChange={(e) => setRegion({ ...region, h: Number(e.target.value) })} style={{ width: "25%" }} />
|
||||
</div>
|
||||
|
||||
<div style={{ marginBottom: "4px" }}>
|
||||
<label>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={visible}
|
||||
onChange={() => setVisible(!visible)}
|
||||
style={{ marginRight: "4px" }}
|
||||
/>
|
||||
Show Overlay on Agent
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<label>Overlay Label:</label>
|
||||
<input
|
||||
type="text"
|
||||
value={alias}
|
||||
onChange={(e) => setAlias(e.target.value)}
|
||||
placeholder="Label (optional)"
|
||||
style={{ width: "100%", fontSize: "9px", marginBottom: "6px" }}
|
||||
/>
|
||||
|
||||
<div style={{ textAlign: "center", fontSize: "8px", color: "#aaa" }}>
|
||||
{imageBase64
|
||||
? `Last image: ${Math.round(imageBase64.length / 1024)} KB`
|
||||
: "Awaiting Screenshot Data..."}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{ position: "absolute", top: 4, right: 4 }}>
|
||||
<IconButton size="small" onClick={handleCopyLiveViewLink}>
|
||||
<ShareIcon style={{ fontSize: 14 }} />
|
||||
</IconButton>
|
||||
</div>
|
||||
<label>Region X / Y / W / H:</label>
|
||||
<div style={{ display: "flex", gap: "4px", marginBottom: "4px" }}>
|
||||
<input type="number" value={region.x} onChange={(e) => {
|
||||
const x = Number(e.target.value);
|
||||
const updated = { ...region, x }; setRegion(updated); regionRef.current = updated;
|
||||
}} style={{ width: "25%" }} />
|
||||
<input type="number" value={region.y} onChange={(e) => {
|
||||
const y = Number(e.target.value);
|
||||
const updated = { ...region, y }; setRegion(updated); regionRef.current = updated;
|
||||
}} style={{ width: "25%" }} />
|
||||
<input type="number" value={region.w} onChange={(e) => {
|
||||
const w = Number(e.target.value);
|
||||
const updated = { ...region, w }; setRegion(updated); regionRef.current = updated;
|
||||
}} style={{ width: "25%" }} />
|
||||
<input type="number" value={region.h} onChange={(e) => {
|
||||
const h = Number(e.target.value);
|
||||
const updated = { ...region, h }; setRegion(updated); regionRef.current = updated;
|
||||
}} style={{ width: "25%" }} />
|
||||
</div>
|
||||
);
|
||||
|
||||
<div style={{ marginBottom: "4px" }}>
|
||||
<label>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={visible}
|
||||
onChange={() => setVisible(!visible)}
|
||||
style={{ marginRight: "4px" }}
|
||||
/>
|
||||
Show Overlay on Agent
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<label>Overlay Label:</label>
|
||||
<input
|
||||
type="text"
|
||||
value={alias}
|
||||
onChange={(e) => setAlias(e.target.value)}
|
||||
placeholder="Label (optional)"
|
||||
style={{ width: "100%", fontSize: "9px", marginBottom: "6px" }}
|
||||
/>
|
||||
|
||||
<div style={{ textAlign: "center", fontSize: "8px", color: "#aaa" }}>
|
||||
{imageBase64
|
||||
? `Last image: ${Math.round(imageBase64.length / 1024)} KB`
|
||||
: "Awaiting Screenshot Data..."}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{ position: "absolute", top: 4, right: 4 }}>
|
||||
<IconButton size="small" onClick={handleCopyLiveViewLink}>
|
||||
<ShareIcon style={{ fontSize: 14 }} />
|
||||
</IconButton>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default {
|
||||
type: "Agent_Role_Screenshot",
|
||||
label: "Agent Role: Screenshot",
|
||||
description: `
|
||||
type: "Agent_Role_Screenshot",
|
||||
label: "Agent Role: Screenshot",
|
||||
description: `
|
||||
Agent Role Node: Screenshot Region
|
||||
|
||||
- Defines a single region capture role
|
||||
- Allows custom update interval and overlay
|
||||
- Emits captured base64 PNG data from agent
|
||||
`.trim(),
|
||||
content: "Capture screenshot region via agent",
|
||||
component: ScreenshotInstructionNode
|
||||
content: "Capture screenshot region via agent",
|
||||
component: ScreenshotInstructionNode
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user