mirror of
https://github.com/bunny-lab-io/Borealis.git
synced 2025-07-27 05:28:29 -06:00
Agent Multi-Role Milestone 2
This commit is contained in:
@ -36,7 +36,6 @@ class ConfigManager:
|
|||||||
self.load()
|
self.load()
|
||||||
|
|
||||||
def load(self):
|
def load(self):
|
||||||
# load or initialize
|
|
||||||
if not os.path.exists(self.path):
|
if not os.path.exists(self.path):
|
||||||
self.data = DEFAULT_CONFIG.copy()
|
self.data = DEFAULT_CONFIG.copy()
|
||||||
self._write()
|
self._write()
|
||||||
@ -44,12 +43,10 @@ class ConfigManager:
|
|||||||
try:
|
try:
|
||||||
with open(self.path, 'r') as f:
|
with open(self.path, 'r') as f:
|
||||||
loaded = json.load(f)
|
loaded = json.load(f)
|
||||||
# merge defaults
|
|
||||||
self.data = {**DEFAULT_CONFIG, **loaded}
|
self.data = {**DEFAULT_CONFIG, **loaded}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[WARN] Failed to parse config: {e}")
|
print(f"[WARN] Failed to parse config: {e}")
|
||||||
self.data = DEFAULT_CONFIG.copy()
|
self.data = DEFAULT_CONFIG.copy()
|
||||||
# track mtime
|
|
||||||
try:
|
try:
|
||||||
self._last_mtime = os.path.getmtime(self.path)
|
self._last_mtime = os.path.getmtime(self.path)
|
||||||
except Exception:
|
except Exception:
|
||||||
@ -74,14 +71,12 @@ class ConfigManager:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
CONFIG = ConfigManager(CONFIG_PATH)
|
CONFIG = ConfigManager(CONFIG_PATH)
|
||||||
# Purge saved regions on startup (fresh run)
|
|
||||||
CONFIG.data['regions'] = {}
|
CONFIG.data['regions'] = {}
|
||||||
CONFIG._write()
|
CONFIG._write()
|
||||||
# //////////////////////////////////////////////////////////////////////////
|
# //////////////////////////////////////////////////////////////////////////
|
||||||
# END CORE SECTION: CONFIG MANAGER
|
# END CORE SECTION: CONFIG MANAGER
|
||||||
# //////////////////////////////////////////////////////////////////////////
|
# //////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
# Assign or persist agent_id
|
|
||||||
host = socket.gethostname().lower()
|
host = socket.gethostname().lower()
|
||||||
stored_id = CONFIG.data.get('agent_id')
|
stored_id = CONFIG.data.get('agent_id')
|
||||||
if stored_id:
|
if stored_id:
|
||||||
@ -108,7 +103,6 @@ async def connect():
|
|||||||
@sio.event
|
@sio.event
|
||||||
async def disconnect():
|
async def disconnect():
|
||||||
print("[WebSocket] Disconnected from Borealis server.")
|
print("[WebSocket] Disconnected from Borealis server.")
|
||||||
# reset tasks and overlays
|
|
||||||
for task in list(role_tasks.values()):
|
for task in list(role_tasks.values()):
|
||||||
task.cancel()
|
task.cancel()
|
||||||
role_tasks.clear()
|
role_tasks.clear()
|
||||||
@ -116,50 +110,50 @@ async def disconnect():
|
|||||||
try: widget.close()
|
try: widget.close()
|
||||||
except: pass
|
except: pass
|
||||||
overlay_widgets.clear()
|
overlay_widgets.clear()
|
||||||
# purge regions on intentional disconnect
|
|
||||||
CONFIG.data['regions'].clear()
|
CONFIG.data['regions'].clear()
|
||||||
CONFIG._write()
|
CONFIG._write()
|
||||||
# reload settings
|
|
||||||
CONFIG.load()
|
CONFIG.load()
|
||||||
|
|
||||||
@sio.on('agent_config')
|
@sio.on('agent_config')
|
||||||
async def on_agent_config(cfg):
|
async def on_agent_config(cfg):
|
||||||
print(f"[CONNECTED] Received config with {len(cfg.get('roles',[]))} roles.")
|
print(f"[CONNECTED] Received config with {len(cfg.get('roles',[]))} roles.")
|
||||||
# determine removed roles
|
|
||||||
new_ids = {r.get('node_id') for r in cfg.get('roles', []) if r.get('node_id')}
|
new_ids = {r.get('node_id') for r in cfg.get('roles', []) if r.get('node_id')}
|
||||||
old_ids = set(role_tasks.keys())
|
old_ids = set(role_tasks.keys())
|
||||||
removed = old_ids - new_ids
|
removed = old_ids - new_ids
|
||||||
|
|
||||||
|
# Cancel removed roles
|
||||||
for rid in removed:
|
for rid in removed:
|
||||||
# remove region config
|
|
||||||
if rid in CONFIG.data['regions']:
|
if rid in CONFIG.data['regions']:
|
||||||
CONFIG.data['regions'].pop(rid, None)
|
CONFIG.data['regions'].pop(rid, None)
|
||||||
# close overlay
|
|
||||||
w = overlay_widgets.pop(rid, None)
|
w = overlay_widgets.pop(rid, None)
|
||||||
if w:
|
if w:
|
||||||
try: w.close()
|
try: w.close()
|
||||||
except: pass
|
except: pass
|
||||||
|
|
||||||
if removed:
|
if removed:
|
||||||
CONFIG._write()
|
CONFIG._write()
|
||||||
# cancel existing and start new
|
|
||||||
|
# Cancel all existing to ensure clean state
|
||||||
for task in list(role_tasks.values()):
|
for task in list(role_tasks.values()):
|
||||||
task.cancel()
|
task.cancel()
|
||||||
role_tasks.clear()
|
role_tasks.clear()
|
||||||
|
|
||||||
|
# Restart everything to ensure roles are re-applied
|
||||||
for role_cfg in cfg.get('roles', []):
|
for role_cfg in cfg.get('roles', []):
|
||||||
|
nid = role_cfg.get('node_id')
|
||||||
if role_cfg.get('role') == 'screenshot':
|
if role_cfg.get('role') == 'screenshot':
|
||||||
nid = role_cfg.get('node_id')
|
task = asyncio.create_task(screenshot_task(role_cfg))
|
||||||
t = asyncio.create_task(screenshot_task(role_cfg))
|
role_tasks[nid] = task
|
||||||
role_tasks[nid] = t
|
|
||||||
# //////////////////////////////////////////////////////////////////////////
|
# //////////////////////////////////////////////////////////////////////////
|
||||||
# END CORE SECTION: WEBSOCKET SETUP & HANDLERS
|
# END CORE SECTION: WEBSOCKET SETUP & HANDLERS
|
||||||
# //////////////////////////////////////////////////////////////////////////
|
# //////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
# ---------------- Overlay Widget ----------------
|
|
||||||
class ScreenshotRegion(QtWidgets.QWidget):
|
class ScreenshotRegion(QtWidgets.QWidget):
|
||||||
def __init__(self, node_id, x=100, y=100, w=300, h=200):
|
def __init__(self, node_id, x=100, y=100, w=300, h=200):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.node_id = node_id
|
self.node_id = node_id
|
||||||
self.setGeometry(x, y, w, h)
|
self.setGeometry(x, y, w, h)
|
||||||
self.setWindowFlags(QtCore.Qt.FramelessWindowHint|QtCore.Qt.WindowStaysOnTopHint)
|
self.setWindowFlags(QtCore.Qt.FramelessWindowHint | QtCore.Qt.WindowStaysOnTopHint)
|
||||||
self.setAttribute(QtCore.Qt.WA_TranslucentBackground)
|
self.setAttribute(QtCore.Qt.WA_TranslucentBackground)
|
||||||
self.drag_offset = None
|
self.drag_offset = None
|
||||||
self.resizing = False
|
self.resizing = False
|
||||||
@ -168,7 +162,7 @@ class ScreenshotRegion(QtWidgets.QWidget):
|
|||||||
self.label = QtWidgets.QLabel(self)
|
self.label = QtWidgets.QLabel(self)
|
||||||
self.label.setText(f"{node_id[:8]}")
|
self.label.setText(f"{node_id[:8]}")
|
||||||
self.label.setStyleSheet("color: lime; background: transparent; font-size: 10px;")
|
self.label.setStyleSheet("color: lime; background: transparent; font-size: 10px;")
|
||||||
self.label.move(8,4)
|
self.label.move(8, 4)
|
||||||
self.setMouseTracking(True)
|
self.setMouseTracking(True)
|
||||||
|
|
||||||
def paintEvent(self, event):
|
def paintEvent(self, event):
|
||||||
@ -183,51 +177,72 @@ class ScreenshotRegion(QtWidgets.QWidget):
|
|||||||
|
|
||||||
def mousePressEvent(self, e):
|
def mousePressEvent(self, e):
|
||||||
if e.button()==QtCore.Qt.LeftButton:
|
if e.button()==QtCore.Qt.LeftButton:
|
||||||
x,y = e.pos().x(), e.pos().y()
|
x, y = e.pos().x(), e.pos().y()
|
||||||
if x>self.width()-self.resize_handle_size and y>self.height()-self.resize_handle_size:
|
if x > self.width() - self.resize_handle_size and y > self.height() - self.resize_handle_size:
|
||||||
self.resizing=True
|
self.resizing = True
|
||||||
else:
|
else:
|
||||||
self.drag_offset = e.globalPos() - self.frameGeometry().topLeft()
|
self.drag_offset = e.globalPos() - self.frameGeometry().topLeft()
|
||||||
|
|
||||||
def mouseMoveEvent(self, e):
|
def mouseMoveEvent(self, e):
|
||||||
if self.resizing:
|
if self.resizing:
|
||||||
nw = max(e.pos().x(),100)
|
nw = max(e.pos().x(), 100)
|
||||||
nh = max(e.pos().y(),80)
|
nh = max(e.pos().y(), 80)
|
||||||
self.resize(nw,nh)
|
self.resize(nw, nh)
|
||||||
elif e.buttons()&QtCore.Qt.LeftButton and self.drag_offset:
|
elif e.buttons() & QtCore.Qt.LeftButton and self.drag_offset:
|
||||||
self.move(e.globalPos()-self.drag_offset)
|
self.move(e.globalPos() - self.drag_offset)
|
||||||
|
|
||||||
def mouseReleaseEvent(self,e):
|
def mouseReleaseEvent(self, e):
|
||||||
self.resizing=False; self.drag_offset=None
|
self.resizing = False
|
||||||
|
self.drag_offset = None
|
||||||
|
|
||||||
def get_geometry(self):
|
def get_geometry(self):
|
||||||
g=self.geometry(); return (g.x(),g.y(),g.width(),g.height())
|
g = self.geometry()
|
||||||
|
return (g.x(), g.y(), g.width(), g.height())
|
||||||
|
|
||||||
# ---------------- Screenshot Task ----------------
|
# ---------------- Screenshot Task ----------------
|
||||||
async def screenshot_task(cfg):
|
async def screenshot_task(cfg):
|
||||||
nid = cfg.get('node_id');
|
nid = cfg.get('node_id')
|
||||||
# initial region from config or payload
|
# If existing region in config, honor that
|
||||||
r = CONFIG.data['regions'].get(nid)
|
r = CONFIG.data['regions'].get(nid)
|
||||||
region = (r['x'],r['y'],r['w'],r['h']) if r else (cfg.get('x',100),cfg.get('y',100),cfg.get('w',300),cfg.get('h',200))
|
if r:
|
||||||
|
region = (r['x'], r['y'], r['w'], r['h'])
|
||||||
|
else:
|
||||||
|
region = (cfg.get('x', 100), cfg.get('y', 100), cfg.get('w', 300), cfg.get('h', 200))
|
||||||
|
CONFIG.data['regions'][nid] = {
|
||||||
|
'x': region[0], 'y': region[1], 'w': region[2], 'h': region[3]
|
||||||
|
}
|
||||||
|
CONFIG._write()
|
||||||
|
|
||||||
if nid not in overlay_widgets:
|
if nid not in overlay_widgets:
|
||||||
widget = ScreenshotRegion(nid,*region)
|
widget = ScreenshotRegion(nid, *region)
|
||||||
overlay_widgets[nid] = widget; widget.show()
|
overlay_widgets[nid] = widget
|
||||||
interval = cfg.get('interval',1000)/1000.0
|
widget.show()
|
||||||
|
|
||||||
|
interval = cfg.get('interval', 1000) / 1000.0
|
||||||
loop = asyncio.get_event_loop()
|
loop = asyncio.get_event_loop()
|
||||||
executor = concurrent.futures.ThreadPoolExecutor(max_workers=CONFIG.data.get('max_task_workers',DEFAULT_CONFIG['max_task_workers']))
|
executor = concurrent.futures.ThreadPoolExecutor(
|
||||||
|
max_workers=CONFIG.data.get('max_task_workers', DEFAULT_CONFIG['max_task_workers'])
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
while True:
|
while True:
|
||||||
x,y,w,h = overlay_widgets[nid].get_geometry()
|
x, y, w, h = overlay_widgets[nid].get_geometry()
|
||||||
# persist if changed
|
|
||||||
prev = CONFIG.data['regions'].get(nid)
|
prev = CONFIG.data['regions'].get(nid)
|
||||||
if prev!={'x':x,'y':y,'w':w,'h':h}:
|
new_geom = {'x': x, 'y': y, 'w': w, 'h': h}
|
||||||
CONFIG.data['regions'][nid]={'x':x,'y':y,'w':w,'h':h}
|
if prev != new_geom:
|
||||||
|
CONFIG.data['regions'][nid] = new_geom
|
||||||
CONFIG._write()
|
CONFIG._write()
|
||||||
grab = partial(ImageGrab.grab,bbox=(x,y,x+w,y+h))
|
grab = partial(ImageGrab.grab, bbox=(x, y, x+w, y+h))
|
||||||
img = await loop.run_in_executor(executor,grab)
|
img = await loop.run_in_executor(executor, grab)
|
||||||
buf = BytesIO(); img.save(buf,format='PNG')
|
buf = BytesIO()
|
||||||
|
img.save(buf, format='PNG')
|
||||||
encoded = base64.b64encode(buf.getvalue()).decode('utf-8')
|
encoded = base64.b64encode(buf.getvalue()).decode('utf-8')
|
||||||
await sio.emit('agent_screenshot_task',{'agent_id':AGENT_ID,'node_id':nid,'image_base64':encoded})
|
await sio.emit('agent_screenshot_task', {
|
||||||
|
'agent_id': AGENT_ID,
|
||||||
|
'node_id': nid,
|
||||||
|
'image_base64': encoded,
|
||||||
|
'x': x, 'y': y, 'w': w, 'h': h # Bi-directional live-sync
|
||||||
|
})
|
||||||
await asyncio.sleep(interval)
|
await asyncio.sleep(interval)
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
return
|
return
|
||||||
@ -238,24 +253,24 @@ async def screenshot_task(cfg):
|
|||||||
async def config_watcher():
|
async def config_watcher():
|
||||||
while True:
|
while True:
|
||||||
if CONFIG.watch(): pass
|
if CONFIG.watch(): pass
|
||||||
await asyncio.sleep(CONFIG.data.get('config_file_watcher_interval',DEFAULT_CONFIG['config_file_watcher_interval']))
|
await asyncio.sleep(CONFIG.data.get('config_file_watcher_interval', DEFAULT_CONFIG['config_file_watcher_interval']))
|
||||||
|
|
||||||
# //////////////////////////////////////////////////////////////////////////
|
# //////////////////////////////////////////////////////////////////////////
|
||||||
# CORE SECTION: MAIN & EVENT LOOP (do not modify unless you know what you’re doing)
|
# CORE SECTION: MAIN & EVENT LOOP (do not modify unless you know what you’re doing)
|
||||||
# //////////////////////////////////////////////////////////////////////////
|
# //////////////////////////////////////////////////////////////////////////
|
||||||
async def connect_loop():
|
async def connect_loop():
|
||||||
retry=5
|
retry = 5
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
url=CONFIG.data.get('borealis_server_url',DEFAULT_CONFIG['borealis_server_url'])
|
url = CONFIG.data.get('borealis_server_url', DEFAULT_CONFIG['borealis_server_url'])
|
||||||
print(f"[WebSocket] Connecting to {url}...")
|
print(f"[WebSocket] Connecting to {url}...")
|
||||||
await sio.connect(url,transports=['websocket'])
|
await sio.connect(url, transports=['websocket'])
|
||||||
break
|
break
|
||||||
except:
|
except:
|
||||||
print(f"[WebSocket] Server unavailable, retrying in {retry}s...")
|
print(f"[WebSocket] Server unavailable, retrying in {retry}s...")
|
||||||
await asyncio.sleep(retry)
|
await asyncio.sleep(retry)
|
||||||
|
|
||||||
if __name__=='__main__':
|
if __name__ == '__main__':
|
||||||
app = QtWidgets.QApplication(sys.argv)
|
app = QtWidgets.QApplication(sys.argv)
|
||||||
loop = QEventLoop(app)
|
loop = QEventLoop(app)
|
||||||
asyncio.set_event_loop(loop)
|
asyncio.set_event_loop(loop)
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/WebUI/src/nodes/Agent/Node_Agent.jsx
|
////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/WebUI/src/nodes/Agent/Node_Agent.jsx
|
||||||
|
|
||||||
import React, { useEffect, useState, useCallback, useMemo } from "react";
|
import React, { useEffect, useState, useCallback, useMemo, useRef } from "react";
|
||||||
import { Handle, Position, useReactFlow, useStore } from "reactflow";
|
import { Handle, Position, useReactFlow, useStore } from "reactflow";
|
||||||
|
|
||||||
const BorealisAgentNode = ({ id, data }) => {
|
const BorealisAgentNode = ({ id, data }) => {
|
||||||
@ -9,18 +9,20 @@ const BorealisAgentNode = ({ id, data }) => {
|
|||||||
const [agents, setAgents] = useState({});
|
const [agents, setAgents] = useState({});
|
||||||
const [selectedAgent, setSelectedAgent] = useState(data.agent_id || "");
|
const [selectedAgent, setSelectedAgent] = useState(data.agent_id || "");
|
||||||
const [isConnected, setIsConnected] = useState(false);
|
const [isConnected, setIsConnected] = useState(false);
|
||||||
|
const prevRolesRef = useRef([]);
|
||||||
|
|
||||||
// Build a normalized list [{id, status}, ...]
|
|
||||||
const agentList = useMemo(() => {
|
const agentList = useMemo(() => {
|
||||||
if (Array.isArray(agents)) {
|
if (!agents || typeof agents !== "object") return [];
|
||||||
return agents.map((a) => ({ id: a.id, status: a.status }));
|
return Object.entries(agents)
|
||||||
} else if (agents && typeof agents === "object") {
|
.map(([aid, info]) => ({
|
||||||
return Object.entries(agents).map(([aid, info]) => ({ id: aid, status: info.status }));
|
id: aid,
|
||||||
}
|
status: info?.status || "offline",
|
||||||
return [];
|
last_seen: info?.last_seen || 0
|
||||||
|
}))
|
||||||
|
.filter(({ status }) => status !== "offline")
|
||||||
|
.sort((a, b) => b.last_seen - a.last_seen);
|
||||||
}, [agents]);
|
}, [agents]);
|
||||||
|
|
||||||
// Fetch agents from backend
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const fetchAgents = () => {
|
const fetchAgents = () => {
|
||||||
fetch("/api/agents")
|
fetch("/api/agents")
|
||||||
@ -29,11 +31,10 @@ const BorealisAgentNode = ({ id, data }) => {
|
|||||||
.catch(() => {});
|
.catch(() => {});
|
||||||
};
|
};
|
||||||
fetchAgents();
|
fetchAgents();
|
||||||
const interval = setInterval(fetchAgents, 5000);
|
const interval = setInterval(fetchAgents, 4000);
|
||||||
return () => clearInterval(interval);
|
return () => clearInterval(interval);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
// Persist selectedAgent and reset connection on change
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setNodes((nds) =>
|
setNodes((nds) =>
|
||||||
nds.map((n) =>
|
nds.map((n) =>
|
||||||
@ -43,7 +44,6 @@ const BorealisAgentNode = ({ id, data }) => {
|
|||||||
setIsConnected(false);
|
setIsConnected(false);
|
||||||
}, [selectedAgent]);
|
}, [selectedAgent]);
|
||||||
|
|
||||||
// Compute attached role node IDs for this agent node
|
|
||||||
const attachedRoleIds = useMemo(
|
const attachedRoleIds = useMemo(
|
||||||
() =>
|
() =>
|
||||||
edges
|
edges
|
||||||
@ -52,7 +52,6 @@ const BorealisAgentNode = ({ id, data }) => {
|
|||||||
[edges, id]
|
[edges, id]
|
||||||
);
|
);
|
||||||
|
|
||||||
// Build role payloads using the instruction registry
|
|
||||||
const getAttachedRoles = useCallback(() => {
|
const getAttachedRoles = useCallback(() => {
|
||||||
const allNodes = getNodes();
|
const allNodes = getNodes();
|
||||||
return attachedRoleIds
|
return attachedRoleIds
|
||||||
@ -63,42 +62,54 @@ const BorealisAgentNode = ({ id, data }) => {
|
|||||||
.filter((r) => r);
|
.filter((r) => r);
|
||||||
}, [attachedRoleIds, getNodes]);
|
}, [attachedRoleIds, getNodes]);
|
||||||
|
|
||||||
// Connect: send roles to server
|
const provisionRoles = useCallback((roles) => {
|
||||||
const handleConnect = useCallback(() => {
|
if (!selectedAgent || roles.length === 0) return;
|
||||||
if (!selectedAgent) return;
|
|
||||||
const roles = getAttachedRoles();
|
|
||||||
fetch("/api/agent/provision", {
|
fetch("/api/agent/provision", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "Content-Type": "application/json" },
|
headers: { "Content-Type": "application/json" },
|
||||||
body: JSON.stringify({ agent_id: selectedAgent, roles }),
|
body: JSON.stringify({ agent_id: selectedAgent, roles })
|
||||||
})
|
})
|
||||||
.then(() => setIsConnected(true))
|
.then(() => {
|
||||||
|
setIsConnected(true);
|
||||||
|
prevRolesRef.current = roles;
|
||||||
|
})
|
||||||
.catch(() => {});
|
.catch(() => {});
|
||||||
}, [selectedAgent, getAttachedRoles]);
|
}, [selectedAgent]);
|
||||||
|
|
||||||
|
const handleConnect = useCallback(() => {
|
||||||
|
const roles = getAttachedRoles();
|
||||||
|
provisionRoles(roles);
|
||||||
|
}, [getAttachedRoles, provisionRoles]);
|
||||||
|
|
||||||
// Disconnect: clear roles on server
|
|
||||||
const handleDisconnect = useCallback(() => {
|
const handleDisconnect = useCallback(() => {
|
||||||
if (!selectedAgent) return;
|
if (!selectedAgent) return;
|
||||||
fetch("/api/agent/provision", {
|
fetch("/api/agent/provision", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "Content-Type": "application/json" },
|
headers: { "Content-Type": "application/json" },
|
||||||
body: JSON.stringify({ agent_id: selectedAgent, roles: [] }),
|
body: JSON.stringify({ agent_id: selectedAgent, roles: [] })
|
||||||
})
|
})
|
||||||
.then(() => setIsConnected(false))
|
.then(() => {
|
||||||
|
setIsConnected(false);
|
||||||
|
prevRolesRef.current = [];
|
||||||
|
})
|
||||||
.catch(() => {});
|
.catch(() => {});
|
||||||
}, [selectedAgent]);
|
}, [selectedAgent]);
|
||||||
|
|
||||||
// Hot-update roles when attachedRoleIds change
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (isConnected) {
|
const newRoles = getAttachedRoles();
|
||||||
const roles = getAttachedRoles();
|
const prevSerialized = JSON.stringify(prevRolesRef.current || []);
|
||||||
fetch("/api/agent/provision", {
|
const newSerialized = JSON.stringify(newRoles);
|
||||||
method: "POST",
|
if (isConnected && newSerialized !== prevSerialized) {
|
||||||
headers: { "Content-Type": "application/json" },
|
provisionRoles(newRoles);
|
||||||
body: JSON.stringify({ agent_id: selectedAgent, roles }),
|
|
||||||
}).catch(() => {});
|
|
||||||
}
|
}
|
||||||
}, [attachedRoleIds]);
|
}, [attachedRoleIds, isConnected]);
|
||||||
|
|
||||||
|
const selectedAgentStatus = useMemo(() => {
|
||||||
|
if (!selectedAgent) return "Unassigned";
|
||||||
|
const agent = agents[selectedAgent];
|
||||||
|
if (!agent) return "Reconnecting...";
|
||||||
|
return agent.status === "provisioned" ? "Connected" : "Available";
|
||||||
|
}, [agents, selectedAgent]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="borealis-node">
|
<div className="borealis-node">
|
||||||
@ -119,14 +130,11 @@ const BorealisAgentNode = ({ id, data }) => {
|
|||||||
style={{ width: "100%", marginBottom: "6px", fontSize: "9px" }}
|
style={{ width: "100%", marginBottom: "6px", fontSize: "9px" }}
|
||||||
>
|
>
|
||||||
<option value="">-- Select --</option>
|
<option value="">-- Select --</option>
|
||||||
{agentList.map(({ id: aid, status }) => {
|
{agentList.map(({ id: aid, status }) => (
|
||||||
const labelText = status === "provisioned" ? "(Connected)" : "(Ready to Connect)";
|
<option key={aid} value={aid}>
|
||||||
return (
|
{aid} ({status})
|
||||||
<option key={aid} value={aid}>
|
</option>
|
||||||
{aid} {labelText}
|
))}
|
||||||
</option>
|
|
||||||
);
|
|
||||||
})}
|
|
||||||
</select>
|
</select>
|
||||||
|
|
||||||
{isConnected ? (
|
{isConnected ? (
|
||||||
@ -149,7 +157,9 @@ const BorealisAgentNode = ({ id, data }) => {
|
|||||||
<hr style={{ margin: "6px 0", borderColor: "#444" }} />
|
<hr style={{ margin: "6px 0", borderColor: "#444" }} />
|
||||||
|
|
||||||
<div style={{ fontSize: "8px", color: "#aaa" }}>
|
<div style={{ fontSize: "8px", color: "#aaa" }}>
|
||||||
Attach <strong>Agent Role Nodes</strong> to define roles for this agent. Roles will be provisioned automatically.
|
Status: <strong>{selectedAgentStatus}</strong>
|
||||||
|
<br />
|
||||||
|
Attach <strong>Agent Role Nodes</strong> to define live behavior.
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
@ -6,181 +6,194 @@ import ShareIcon from "@mui/icons-material/Share";
|
|||||||
import IconButton from "@mui/material/IconButton";
|
import IconButton from "@mui/material/IconButton";
|
||||||
|
|
||||||
if (!window.BorealisValueBus) {
|
if (!window.BorealisValueBus) {
|
||||||
window.BorealisValueBus = {};
|
window.BorealisValueBus = {};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!window.BorealisUpdateRate) {
|
if (!window.BorealisUpdateRate) {
|
||||||
window.BorealisUpdateRate = 100;
|
window.BorealisUpdateRate = 100;
|
||||||
}
|
}
|
||||||
|
|
||||||
const ScreenshotInstructionNode = ({ id, data }) => {
|
const ScreenshotInstructionNode = ({ id, data }) => {
|
||||||
const { setNodes, getNodes } = useReactFlow();
|
const { setNodes, getNodes } = useReactFlow();
|
||||||
const edges = useStore(state => state.edges);
|
const edges = useStore(state => state.edges);
|
||||||
|
|
||||||
const [interval, setInterval] = useState(data?.interval || 1000);
|
const [interval, setInterval] = useState(data?.interval || 1000);
|
||||||
const [region, setRegion] = useState({
|
const [region, setRegion] = useState({
|
||||||
x: data?.x ?? 250,
|
x: data?.x ?? 250,
|
||||||
y: data?.y ?? 100,
|
y: data?.y ?? 100,
|
||||||
w: data?.w ?? 300,
|
w: data?.w ?? 300,
|
||||||
h: data?.h ?? 200,
|
h: data?.h ?? 200,
|
||||||
});
|
});
|
||||||
const [visible, setVisible] = useState(data?.visible ?? true);
|
const [visible, setVisible] = useState(data?.visible ?? true);
|
||||||
const [alias, setAlias] = useState(data?.alias || "");
|
const [alias, setAlias] = useState(data?.alias || "");
|
||||||
const [imageBase64, setImageBase64] = useState("");
|
const [imageBase64, setImageBase64] = useState("");
|
||||||
|
|
||||||
const base64Ref = useRef("");
|
const base64Ref = useRef("");
|
||||||
|
const regionRef = useRef(region);
|
||||||
|
|
||||||
const handleCopyLiveViewLink = () => {
|
// Push current state into BorealisValueBus at intervals
|
||||||
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner");
|
useEffect(() => {
|
||||||
if (!agentEdge) {
|
const intervalId = setInterval(() => {
|
||||||
alert("No upstream agent connection found.");
|
const val = base64Ref.current;
|
||||||
return;
|
if (val) {
|
||||||
|
window.BorealisValueBus[id] = val;
|
||||||
|
setNodes(nds =>
|
||||||
|
nds.map(n =>
|
||||||
|
n.id === id ? { ...n, data: { ...n.data, value: val } } : n
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}, window.BorealisUpdateRate || 100);
|
||||||
|
return () => clearInterval(intervalId);
|
||||||
|
}, [id, setNodes]);
|
||||||
|
|
||||||
|
// Listen for agent screenshot + overlay updates
|
||||||
|
useEffect(() => {
|
||||||
|
const socket = window.BorealisSocket;
|
||||||
|
if (!socket) return;
|
||||||
|
|
||||||
|
const handleScreenshot = (payload) => {
|
||||||
|
if (payload?.node_id !== id || !payload.image_base64) return;
|
||||||
|
|
||||||
|
base64Ref.current = payload.image_base64;
|
||||||
|
setImageBase64(payload.image_base64);
|
||||||
|
window.BorealisValueBus[id] = payload.image_base64;
|
||||||
|
|
||||||
|
// If geometry changed from agent side, sync into UI
|
||||||
|
const { x, y, w, h } = payload;
|
||||||
|
if (x !== undefined && y !== undefined && w !== undefined && h !== undefined) {
|
||||||
|
const newRegion = { x, y, w, h };
|
||||||
|
const prev = regionRef.current;
|
||||||
|
const changed = Object.entries(newRegion).some(([k, v]) => prev[k] !== v);
|
||||||
|
|
||||||
|
if (changed) {
|
||||||
|
regionRef.current = newRegion;
|
||||||
|
setRegion(newRegion);
|
||||||
|
setNodes(nds =>
|
||||||
|
nds.map(n =>
|
||||||
|
n.id === id ? { ...n, data: { ...n.data, ...newRegion } } : n
|
||||||
|
)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
const agentNode = getNodes().find(n => n.id === agentEdge.source);
|
|
||||||
const selectedAgentId = agentNode?.data?.agent_id;
|
|
||||||
|
|
||||||
if (!selectedAgentId) {
|
|
||||||
alert("Upstream agent node does not have a selected agent.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const liveUrl = `${window.location.origin}/api/agent/${selectedAgentId}/node/${id}/screenshot/live`;
|
|
||||||
navigator.clipboard.writeText(liveUrl)
|
|
||||||
.then(() => console.log(`[Clipboard] Copied Live View URL: ${liveUrl}`))
|
|
||||||
.catch(err => console.error("Clipboard copy failed:", err));
|
|
||||||
};
|
};
|
||||||
|
|
||||||
useEffect(() => {
|
socket.on("agent_screenshot_task", handleScreenshot);
|
||||||
const intervalId = setInterval(() => {
|
return () => socket.off("agent_screenshot_task", handleScreenshot);
|
||||||
const val = base64Ref.current;
|
}, [id, setNodes]);
|
||||||
|
|
||||||
console.log(`[Screenshot Node] setInterval update. Current base64 length: ${val?.length || 0}`);
|
// Bi-directional instruction export
|
||||||
|
window.__BorealisInstructionNodes = window.__BorealisInstructionNodes || {};
|
||||||
|
window.__BorealisInstructionNodes[id] = () => ({
|
||||||
|
node_id: id,
|
||||||
|
role: "screenshot",
|
||||||
|
interval,
|
||||||
|
visible,
|
||||||
|
alias,
|
||||||
|
...regionRef.current
|
||||||
|
});
|
||||||
|
|
||||||
if (!val) return;
|
// Manual live view copy
|
||||||
|
const handleCopyLiveViewLink = () => {
|
||||||
|
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner");
|
||||||
|
const agentNode = getNodes().find(n => n.id === agentEdge?.source);
|
||||||
|
const selectedAgentId = agentNode?.data?.agent_id;
|
||||||
|
|
||||||
window.BorealisValueBus[id] = val;
|
if (!selectedAgentId) {
|
||||||
setNodes(nds =>
|
alert("No valid agent connection found.");
|
||||||
nds.map(n =>
|
return;
|
||||||
n.id === id
|
}
|
||||||
? { ...n, data: { ...n.data, value: val } }
|
|
||||||
: n
|
|
||||||
)
|
|
||||||
);
|
|
||||||
}, window.BorealisUpdateRate || 100);
|
|
||||||
|
|
||||||
return () => clearInterval(intervalId);
|
const liveUrl = `${window.location.origin}/api/agent/${selectedAgentId}/node/${id}/screenshot/live`;
|
||||||
}, [id, setNodes]);
|
navigator.clipboard.writeText(liveUrl)
|
||||||
|
.then(() => console.log(`[Clipboard] Live View URL copied: ${liveUrl}`))
|
||||||
|
.catch(err => console.error("Clipboard copy failed:", err));
|
||||||
|
};
|
||||||
|
|
||||||
useEffect(() => {
|
return (
|
||||||
const socket = window.BorealisSocket || null;
|
<div className="borealis-node" style={{ position: "relative" }}>
|
||||||
if (!socket) {
|
<Handle type="target" position={Position.Left} className="borealis-handle" />
|
||||||
console.warn("[Screenshot Node] BorealisSocket not available");
|
<Handle type="source" position={Position.Right} className="borealis-handle" />
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`[Screenshot Node] Listening for agent_screenshot_task with node_id: ${id}`);
|
<div className="borealis-node-header">Agent Role: Screenshot</div>
|
||||||
|
<div className="borealis-node-content" style={{ fontSize: "9px" }}>
|
||||||
|
<label>Update Interval (ms):</label>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
min="100"
|
||||||
|
step="100"
|
||||||
|
value={interval}
|
||||||
|
onChange={(e) => setInterval(Number(e.target.value))}
|
||||||
|
style={{ width: "100%", marginBottom: "4px" }}
|
||||||
|
/>
|
||||||
|
|
||||||
const handleScreenshot = (payload) => {
|
<label>Region X / Y / W / H:</label>
|
||||||
console.log("[Screenshot Node] Received payload:", payload);
|
<div style={{ display: "flex", gap: "4px", marginBottom: "4px" }}>
|
||||||
|
<input type="number" value={region.x} onChange={(e) => {
|
||||||
if (payload?.node_id === id && payload?.image_base64) {
|
const x = Number(e.target.value);
|
||||||
base64Ref.current = payload.image_base64;
|
const updated = { ...region, x }; setRegion(updated); regionRef.current = updated;
|
||||||
setImageBase64(payload.image_base64);
|
}} style={{ width: "25%" }} />
|
||||||
window.BorealisValueBus[id] = payload.image_base64;
|
<input type="number" value={region.y} onChange={(e) => {
|
||||||
|
const y = Number(e.target.value);
|
||||||
console.log(`[Screenshot Node] Updated base64Ref and ValueBus for ${id}, length: ${payload.image_base64.length}`);
|
const updated = { ...region, y }; setRegion(updated); regionRef.current = updated;
|
||||||
} else {
|
}} style={{ width: "25%" }} />
|
||||||
console.log(`[Screenshot Node] Ignored payload for mismatched node_id (${payload?.node_id})`);
|
<input type="number" value={region.w} onChange={(e) => {
|
||||||
}
|
const w = Number(e.target.value);
|
||||||
};
|
const updated = { ...region, w }; setRegion(updated); regionRef.current = updated;
|
||||||
|
}} style={{ width: "25%" }} />
|
||||||
socket.on("agent_screenshot_task", handleScreenshot);
|
<input type="number" value={region.h} onChange={(e) => {
|
||||||
return () => socket.off("agent_screenshot_task", handleScreenshot);
|
const h = Number(e.target.value);
|
||||||
}, [id]);
|
const updated = { ...region, h }; setRegion(updated); regionRef.current = updated;
|
||||||
|
}} style={{ width: "25%" }} />
|
||||||
window.__BorealisInstructionNodes = window.__BorealisInstructionNodes || {};
|
|
||||||
window.__BorealisInstructionNodes[id] = () => ({
|
|
||||||
node_id: id,
|
|
||||||
role: "screenshot",
|
|
||||||
interval,
|
|
||||||
visible,
|
|
||||||
alias,
|
|
||||||
...region
|
|
||||||
});
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="borealis-node" style={{ position: "relative" }}>
|
|
||||||
<Handle type="target" position={Position.Left} className="borealis-handle" />
|
|
||||||
<Handle type="source" position={Position.Right} className="borealis-handle" />
|
|
||||||
|
|
||||||
<div className="borealis-node-header">Agent Role: Screenshot</div>
|
|
||||||
<div className="borealis-node-content" style={{ fontSize: "9px" }}>
|
|
||||||
<label>Update Interval (ms):</label>
|
|
||||||
<input
|
|
||||||
type="number"
|
|
||||||
min="100"
|
|
||||||
step="100"
|
|
||||||
value={interval}
|
|
||||||
onChange={(e) => setInterval(Number(e.target.value))}
|
|
||||||
style={{ width: "100%", marginBottom: "4px" }}
|
|
||||||
/>
|
|
||||||
|
|
||||||
<label>Region X / Y / W / H:</label>
|
|
||||||
<div style={{ display: "flex", gap: "4px", marginBottom: "4px" }}>
|
|
||||||
<input type="number" value={region.x} onChange={(e) => setRegion({ ...region, x: Number(e.target.value) })} style={{ width: "25%" }} />
|
|
||||||
<input type="number" value={region.y} onChange={(e) => setRegion({ ...region, y: Number(e.target.value) })} style={{ width: "25%" }} />
|
|
||||||
<input type="number" value={region.w} onChange={(e) => setRegion({ ...region, w: Number(e.target.value) })} style={{ width: "25%" }} />
|
|
||||||
<input type="number" value={region.h} onChange={(e) => setRegion({ ...region, h: Number(e.target.value) })} style={{ width: "25%" }} />
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div style={{ marginBottom: "4px" }}>
|
|
||||||
<label>
|
|
||||||
<input
|
|
||||||
type="checkbox"
|
|
||||||
checked={visible}
|
|
||||||
onChange={() => setVisible(!visible)}
|
|
||||||
style={{ marginRight: "4px" }}
|
|
||||||
/>
|
|
||||||
Show Overlay on Agent
|
|
||||||
</label>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<label>Overlay Label:</label>
|
|
||||||
<input
|
|
||||||
type="text"
|
|
||||||
value={alias}
|
|
||||||
onChange={(e) => setAlias(e.target.value)}
|
|
||||||
placeholder="Label (optional)"
|
|
||||||
style={{ width: "100%", fontSize: "9px", marginBottom: "6px" }}
|
|
||||||
/>
|
|
||||||
|
|
||||||
<div style={{ textAlign: "center", fontSize: "8px", color: "#aaa" }}>
|
|
||||||
{imageBase64
|
|
||||||
? `Last image: ${Math.round(imageBase64.length / 1024)} KB`
|
|
||||||
: "Awaiting Screenshot Data..."}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div style={{ position: "absolute", top: 4, right: 4 }}>
|
|
||||||
<IconButton size="small" onClick={handleCopyLiveViewLink}>
|
|
||||||
<ShareIcon style={{ fontSize: 14 }} />
|
|
||||||
</IconButton>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
);
|
|
||||||
|
<div style={{ marginBottom: "4px" }}>
|
||||||
|
<label>
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={visible}
|
||||||
|
onChange={() => setVisible(!visible)}
|
||||||
|
style={{ marginRight: "4px" }}
|
||||||
|
/>
|
||||||
|
Show Overlay on Agent
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<label>Overlay Label:</label>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={alias}
|
||||||
|
onChange={(e) => setAlias(e.target.value)}
|
||||||
|
placeholder="Label (optional)"
|
||||||
|
style={{ width: "100%", fontSize: "9px", marginBottom: "6px" }}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<div style={{ textAlign: "center", fontSize: "8px", color: "#aaa" }}>
|
||||||
|
{imageBase64
|
||||||
|
? `Last image: ${Math.round(imageBase64.length / 1024)} KB`
|
||||||
|
: "Awaiting Screenshot Data..."}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style={{ position: "absolute", top: 4, right: 4 }}>
|
||||||
|
<IconButton size="small" onClick={handleCopyLiveViewLink}>
|
||||||
|
<ShareIcon style={{ fontSize: 14 }} />
|
||||||
|
</IconButton>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
type: "Agent_Role_Screenshot",
|
type: "Agent_Role_Screenshot",
|
||||||
label: "Agent Role: Screenshot",
|
label: "Agent Role: Screenshot",
|
||||||
description: `
|
description: `
|
||||||
Agent Role Node: Screenshot Region
|
Agent Role Node: Screenshot Region
|
||||||
|
|
||||||
- Defines a single region capture role
|
- Defines a single region capture role
|
||||||
- Allows custom update interval and overlay
|
- Allows custom update interval and overlay
|
||||||
- Emits captured base64 PNG data from agent
|
- Emits captured base64 PNG data from agent
|
||||||
`.trim(),
|
`.trim(),
|
||||||
content: "Capture screenshot region via agent",
|
content: "Capture screenshot region via agent",
|
||||||
component: ScreenshotInstructionNode
|
component: ScreenshotInstructionNode
|
||||||
};
|
};
|
||||||
|
Reference in New Issue
Block a user