bunny-lab/Borealis-Github-Replica
1
0
Fork 0
mirror of https://github.com/bunny-lab-io/Borealis.git synced 2025-10-27 05:01:57 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #92 from bunny-lab-io:codex/fix-agent-screenshot-functionality

Browse Source 
Fix screenshot role config handling
This commit is contained in:
Nicole Rappe
2025-10-15 06:43:08 -06:00
committed by GitHub
parent 7e419c8cb5 91d246a8fa
commit a126cb9faa
6 changed files with 481 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
Data/Agent/Roles/role_Screenshot.py
View File

@@ -44,12 +44,65 @@ extra_top_padding = overlay_green_thickness * 2 + 4
overlay_widgets = {} overlay_widgets = {}
def _coerce_int(value, default, minimum=None, maximum=None):
try:
if value is None:
raise ValueError
if isinstance(value, bool):
raise ValueError
if isinstance(value, (int, float)):
ivalue = int(value)
else:
text = str(value).strip()
if not text:
raise ValueError
ivalue = int(float(text))
if minimum is not None and ivalue < minimum:
return minimum
if maximum is not None and ivalue > maximum:
return maximum
return ivalue
except Exception:
return default
def _coerce_bool(value, default=True):
if isinstance(value, bool):
return value
if isinstance(value, str):
text = value.strip().lower()
if text in {'true', '1', 'yes', 'on'}:
return True
if text in {'false', '0', 'no', 'off'}:
return False
return default
def _coerce_text(value):
if value is None:
return ''
try:
return str(value)
except Exception:
return ''
def _normalize_mode(value):
text = _coerce_text(value).strip().lower()
if text in {'interactive', 'currentuser', 'user'}:
return 'currentuser'
if text in {'system', 'svc', 'service'}:
return 'system'
return ''
class ScreenshotRegion(QtWidgets.QWidget): class ScreenshotRegion(QtWidgets.QWidget):
def __init__(self, ctx, node_id, x=100, y=100, w=300, h=200, alias=None): def __init__(self, ctx, node_id, x=100, y=100, w=300, h=200, alias=None):
super().__init__() super().__init__()
self.ctx = ctx self.ctx = ctx
self.node_id = node_id self.node_id = node_id
self.alias = alias self.alias = _coerce_text(alias)
self._visible = True
self.setGeometry( self.setGeometry(
x - handle_size, x - handle_size,
y - handle_size - extra_top_padding, y - handle_size - extra_top_padding,
@@ -99,6 +152,19 @@ class ScreenshotRegion(QtWidgets.QWidget):
p.setBrush(QtGui.QColor(0, 191, 255)) p.setBrush(QtGui.QColor(0, 191, 255))
p.drawRect(bar_x, bar_y - bar_height - 10, bar_width, bar_height * 4) p.drawRect(bar_x, bar_y - bar_height - 10, bar_width, bar_height * 4)
if self.alias:
p.setPen(QtGui.QPen(QtGui.QColor(255, 255, 255)))
font = QtGui.QFont()
font.setPointSize(10)
p.setFont(font)
text_rect = QtCore.QRect(
overlay_green_thickness * 2,
extra_top_padding + overlay_green_thickness * 2,
w - overlay_green_thickness * 4,
overlay_green_thickness * 10,
)
p.drawText(text_rect, QtCore.Qt.AlignLeft | QtCore.Qt.AlignTop, self.alias)
def get_geometry(self): def get_geometry(self):
g = self.geometry() g = self.geometry()
return ( return (
@@ -108,6 +174,29 @@ class ScreenshotRegion(QtWidgets.QWidget):
g.height() - handle_size * 2 - extra_top_padding, g.height() - handle_size * 2 - extra_top_padding,
) )
def set_region(self, x, y, w, h):
self.setGeometry(
x - handle_size,
y - handle_size - extra_top_padding,
w + handle_size * 2,
h + handle_size * 2 + extra_top_padding,
)
def set_alias(self, alias):
self.alias = _coerce_text(alias)
self.update()
def apply_visibility(self, visible: bool):
self._visible = bool(visible)
if self._visible:
self.show()
try:
self.raise_()
except Exception:
pass
else:
self.hide()
def mousePressEvent(self, e): def mousePressEvent(self, e):
if e.button() == QtCore.Qt.LeftButton: if e.button() == QtCore.Qt.LeftButton:
pos = e.pos() pos = e.pos()
@@ -184,6 +273,7 @@ class Role:
def __init__(self, ctx): def __init__(self, ctx):
self.ctx = ctx self.ctx = ctx
self.tasks = {} self.tasks = {}
self.running_configs = {}
def register_events(self): def register_events(self):
sio = self.ctx.sio sio = self.ctx.sio
@@ -214,6 +304,7 @@ class Role:
except Exception: except Exception:
pass pass
self.tasks.clear() self.tasks.clear()
self.running_configs.clear()
# Close all widgets # Close all widgets
for nid in list(overlay_widgets.keys()): for nid in list(overlay_widgets.keys()):
self._close_overlay(nid) self._close_overlay(nid)
@@ -222,10 +313,16 @@ class Role:
# Filter only screenshot roles # Filter only screenshot roles
screenshot_roles = [r for r in roles_cfg if (r.get('role') == 'screenshot')] screenshot_roles = [r for r in roles_cfg if (r.get('role') == 'screenshot')]
sanitized_roles = []
for rcfg in screenshot_roles:
sanitized = self._normalize_config(rcfg)
if sanitized:
sanitized_roles.append(sanitized)
# Optional: forward interval to SYSTEM helper via hook # Optional: forward interval to SYSTEM helper via hook
try: try:
if screenshot_roles and 'send_service_control' in self.ctx.hooks: if sanitized_roles and 'send_service_control' in self.ctx.hooks:
interval_ms = int(screenshot_roles[0].get('interval', 1000)) interval_ms = sanitized_roles[0]['interval']
try: try:
self.ctx.hooks['send_service_control']({'type': 'screenshot_config', 'interval_ms': interval_ms}) self.ctx.hooks['send_service_control']({'type': 'screenshot_config', 'interval_ms': interval_ms})
except Exception: except Exception:
@@ -234,7 +331,7 @@ class Role:
pass pass
# Cancel tasks that are no longer present # Cancel tasks that are no longer present
new_ids = {r.get('node_id') for r in screenshot_roles if r.get('node_id')} new_ids = {r.get('node_id') for r in sanitized_roles if r.get('node_id')}
old_ids = set(self.tasks.keys()) old_ids = set(self.tasks.keys())
removed = old_ids - new_ids removed = old_ids - new_ids
for rid in removed: for rid in removed:
@@ -250,6 +347,7 @@ class Role:
self._close_overlay(rid) self._close_overlay(rid)
except Exception: except Exception:
pass pass
self.running_configs.pop(rid, None)
if removed: if removed:
try: try:
self.ctx.config._write() self.ctx.config._write()
@@ -257,39 +355,87 @@ class Role:
pass pass
# Start tasks for all screenshot roles in config # Start tasks for all screenshot roles in config
for rcfg in screenshot_roles: for rcfg in sanitized_roles:
nid = rcfg.get('node_id') nid = rcfg.get('node_id')
if not nid: if not nid:
continue continue
if nid in self.tasks: if nid in self.tasks:
continue if self.running_configs.get(nid) == rcfg:
continue
prev = self.tasks.pop(nid)
try:
prev.cancel()
except Exception:
pass
task = asyncio.create_task(self._screenshot_task(rcfg)) task = asyncio.create_task(self._screenshot_task(rcfg))
self.tasks[nid] = task self.tasks[nid] = task
self.running_configs[nid] = rcfg
def _normalize_config(self, cfg):
try:
nid = cfg.get('node_id')
if not nid:
return None
norm = {
'node_id': nid,
'interval': _coerce_int(cfg.get('interval'), 1000, minimum=100),
'x': _coerce_int(cfg.get('x'), 100),
'y': _coerce_int(cfg.get('y'), 100),
'w': _coerce_int(cfg.get('w'), 300, minimum=1),
'h': _coerce_int(cfg.get('h'), 200, minimum=1),
'visible': _coerce_bool(cfg.get('visible'), True),
'alias': _coerce_text(cfg.get('alias')),
'target_agent_mode': _normalize_mode(cfg.get('target_agent_mode')),
'target_agent_host': _coerce_text(cfg.get('target_agent_host')),
}
return norm
except Exception:
return None
async def _screenshot_task(self, cfg): async def _screenshot_task(self, cfg):
cfg = self._normalize_config(cfg) or {}
nid = cfg.get('node_id') nid = cfg.get('node_id')
alias = cfg.get('alias', '') if not nid:
reg = self.ctx.config.data.setdefault('regions', {}) return
r = reg.get(nid)
if r:
region = (r['x'], r['y'], r['w'], r['h'])
else:
region = (
cfg.get('x', 100),
cfg.get('y', 100),
cfg.get('w', 300),
cfg.get('h', 200),
)
reg[nid] = {'x': region[0], 'y': region[1], 'w': region[2], 'h': region[3]}
try:
self.ctx.config._write()
except Exception:
pass
if nid not in overlay_widgets: target_mode = cfg.get('target_agent_mode') or ''
current_mode = getattr(self.ctx, 'service_mode', '') or ''
if target_mode and current_mode and target_mode != current_mode:
return
alias = cfg.get('alias', '')
visible = cfg.get('visible', True)
reg = self.ctx.config.data.setdefault('regions', {})
stored = reg.get(nid) if isinstance(reg.get(nid), dict) else None
base_region = (
cfg.get('x', 100),
cfg.get('y', 100),
cfg.get('w', 300),
cfg.get('h', 200),
)
if stored:
region = (
_coerce_int(stored.get('x'), base_region[0]),
_coerce_int(stored.get('y'), base_region[1]),
_coerce_int(stored.get('w'), base_region[2], minimum=1),
_coerce_int(stored.get('h'), base_region[3], minimum=1),
)
else:
region = base_region
reg[nid] = {'x': region[0], 'y': region[1], 'w': region[2], 'h': region[3]}
try:
self.ctx.config._write()
except Exception:
pass
widget = overlay_widgets.get(nid)
if widget is None:
widget = ScreenshotRegion(self.ctx, nid, *region, alias=alias) widget = ScreenshotRegion(self.ctx, nid, *region, alias=alias)
overlay_widgets[nid] = widget overlay_widgets[nid] = widget
widget.show() else:
widget.set_region(*region)
widget.set_alias(alias)
widget.apply_visibility(visible)
await self.ctx.sio.emit('agent_screenshot_task', { await self.ctx.sio.emit('agent_screenshot_task', {
'agent_id': self.ctx.agent_id, 'agent_id': self.ctx.agent_id,
@@ -298,14 +444,17 @@ class Role:
'x': region[0], 'y': region[1], 'w': region[2], 'h': region[3] 'x': region[0], 'y': region[1], 'w': region[2], 'h': region[3]
}) })
interval = cfg.get('interval', 1000) / 1000.0 interval = max(cfg.get('interval', 1000), 50) / 1000.0
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
# Maximum number of screenshot roles you can assign to an agent. (8 already feels overkill) # Maximum number of screenshot roles you can assign to an agent. (8 already feels overkill)
executor = concurrent.futures.ThreadPoolExecutor(max_workers=8) executor = concurrent.futures.ThreadPoolExecutor(max_workers=8)
try: try:
while True: while True:
x, y, w, h = overlay_widgets[nid].get_geometry() widget = overlay_widgets.get(nid)
if widget is None:
break
x, y, w, h = widget.get_geometry()
grab = partial(ImageGrab.grab, bbox=(x, y, x + w, y + h)) grab = partial(ImageGrab.grab, bbox=(x, y, x + w, y + h))
img = await loop.run_in_executor(executor, grab) img = await loop.run_in_executor(executor, grab)
buf = BytesIO(); img.save(buf, format='PNG') buf = BytesIO(); img.save(buf, format='PNG')
@@ -321,3 +470,8 @@ class Role:
pass pass
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
finally:
try:
executor.shutdown(wait=False)
except Exception:
pass

21
Data/Agent/agent.py
View File

@@ -72,6 +72,7 @@ def _bootstrap_log(msg: str):
# Headless/service mode flag (skip Qt and interactive UI) # Headless/service mode flag (skip Qt and interactive UI)
SYSTEM_SERVICE_MODE = ('--system-service' in sys.argv) or (os.environ.get('BOREALIS_AGENT_MODE') == 'system') SYSTEM_SERVICE_MODE = ('--system-service' in sys.argv) or (os.environ.get('BOREALIS_AGENT_MODE') == 'system')
SERVICE_MODE = 'system' if SYSTEM_SERVICE_MODE else 'currentuser'
_bootstrap_log(f'agent.py loaded; SYSTEM_SERVICE_MODE={SYSTEM_SERVICE_MODE}; argv={sys.argv!r}') _bootstrap_log(f'agent.py loaded; SYSTEM_SERVICE_MODE={SYSTEM_SERVICE_MODE}; argv={sys.argv!r}')
def _argv_get(flag: str, default: str = None): def _argv_get(flag: str, default: str = None):
try: try:
@@ -859,7 +860,8 @@ async def send_heartbeat():
"agent_id": AGENT_ID, "agent_id": AGENT_ID,
"hostname": socket.gethostname(), "hostname": socket.gethostname(),
"agent_operating_system": detect_agent_os(), "agent_operating_system": detect_agent_os(),
"last_seen": int(time.time()) "last_seen": int(time.time()),
"service_mode": SERVICE_MODE,
} }
await sio.emit("agent_heartbeat", payload) await sio.emit("agent_heartbeat", payload)
# Also report collector status alive ping. # Also report collector status alive ping.
@@ -872,6 +874,7 @@ async def send_heartbeat():
'agent_id': AGENT_ID, 'agent_id': AGENT_ID,
'hostname': socket.gethostname(), 'hostname': socket.gethostname(),
'active': True, 'active': True,
'service_mode': SERVICE_MODE,
'last_user': f"{os.environ.get('USERDOMAIN') or socket.gethostname()}\\{getpass.getuser()}" 'last_user': f"{os.environ.get('USERDOMAIN') or socket.gethostname()}\\{getpass.getuser()}"
}) })
else: else:
@@ -879,6 +882,7 @@ async def send_heartbeat():
'agent_id': AGENT_ID, 'agent_id': AGENT_ID,
'hostname': socket.gethostname(), 'hostname': socket.gethostname(),
'active': True, 'active': True,
'service_mode': SERVICE_MODE,
}) })
except Exception: except Exception:
pass pass
@@ -1203,7 +1207,7 @@ async def send_agent_details_once():
async def connect(): async def connect():
print(f"[INFO] Successfully Connected to Borealis Server!") print(f"[INFO] Successfully Connected to Borealis Server!")
_log_agent('Connected to server.') _log_agent('Connected to server.')
await sio.emit('connect_agent', {"agent_id": AGENT_ID}) await sio.emit('connect_agent', {"agent_id": AGENT_ID, "service_mode": SERVICE_MODE})
# Send an immediate heartbeat so the UI can populate instantly. # Send an immediate heartbeat so the UI can populate instantly.
try: try:
@@ -1211,7 +1215,8 @@ async def connect():
"agent_id": AGENT_ID, "agent_id": AGENT_ID,
"hostname": socket.gethostname(), "hostname": socket.gethostname(),
"agent_operating_system": detect_agent_os(), "agent_operating_system": detect_agent_os(),
"last_seen": int(time.time()) "last_seen": int(time.time()),
"service_mode": SERVICE_MODE,
}) })
except Exception as e: except Exception as e:
print(f"[WARN] initial heartbeat failed: {e}") print(f"[WARN] initial heartbeat failed: {e}")
@@ -1225,6 +1230,7 @@ async def connect():
'agent_id': AGENT_ID, 'agent_id': AGENT_ID,
'hostname': socket.gethostname(), 'hostname': socket.gethostname(),
'active': True, 'active': True,
'service_mode': SERVICE_MODE,
'last_user': f"{os.environ.get('USERDOMAIN') or socket.gethostname()}\\{getpass.getuser()}" 'last_user': f"{os.environ.get('USERDOMAIN') or socket.gethostname()}\\{getpass.getuser()}"
}) })
else: else:
@@ -1232,6 +1238,7 @@ async def connect():
'agent_id': AGENT_ID, 'agent_id': AGENT_ID,
'hostname': socket.gethostname(), 'hostname': socket.gethostname(),
'active': True, 'active': True,
'service_mode': SERVICE_MODE,
}) })
except Exception: except Exception:
pass pass
@@ -1542,9 +1549,10 @@ if __name__=='__main__':
# Initialize roles context for role tasks # Initialize roles context for role tasks
# Initialize role manager and hot-load roles from Roles/ # Initialize role manager and hot-load roles from Roles/
try: try:
hooks = {'send_service_control': send_service_control, 'get_server_url': get_server_url} base_hooks = {'send_service_control': send_service_control, 'get_server_url': get_server_url}
if not SYSTEM_SERVICE_MODE: if not SYSTEM_SERVICE_MODE:
# Load interactive-context roles (tray/UI, current-user execution, screenshot, etc.) # Load interactive-context roles (tray/UI, current-user execution, screenshot, etc.)
hooks_interactive = {**base_hooks, 'service_mode': 'currentuser'}
ROLE_MANAGER = RoleManager( ROLE_MANAGER = RoleManager(
base_dir=os.path.dirname(__file__), base_dir=os.path.dirname(__file__),
context='interactive', context='interactive',
@@ -1552,12 +1560,13 @@ if __name__=='__main__':
agent_id=AGENT_ID, agent_id=AGENT_ID,
config=CONFIG, config=CONFIG,
loop=loop, loop=loop,
hooks=hooks, hooks=hooks_interactive,
) )
ROLE_MANAGER.load() ROLE_MANAGER.load()
# Load system roles only when running in SYSTEM service mode # Load system roles only when running in SYSTEM service mode
ROLE_MANAGER_SYS = None ROLE_MANAGER_SYS = None
if SYSTEM_SERVICE_MODE: if SYSTEM_SERVICE_MODE:
hooks_system = {**base_hooks, 'service_mode': 'system'}
ROLE_MANAGER_SYS = RoleManager( ROLE_MANAGER_SYS = RoleManager(
base_dir=os.path.dirname(__file__), base_dir=os.path.dirname(__file__),
context='system', context='system',
@@ -1565,7 +1574,7 @@ if __name__=='__main__':
agent_id=AGENT_ID, agent_id=AGENT_ID,
config=CONFIG, config=CONFIG,
loop=loop, loop=loop,
hooks=hooks, hooks=hooks_system,
) )
ROLE_MANAGER_SYS.load() ROLE_MANAGER_SYS.load()
except Exception as e: except Exception as e:

4
Data/Agent/role_manager.py
View File

@@ -24,6 +24,10 @@ class RoleManager:
self.config = config self.config = config
self.loop = loop self.loop = loop
self.hooks = hooks or {} self.hooks = hooks or {}
try:
self.service_mode = (self.hooks.get('service_mode') or '').strip().lower()
except Exception:
self.service_mode = ''
def __init__(self, base_dir: str, context: str, sio, agent_id: str, config, loop, hooks: Optional[dict] = None): def __init__(self, base_dir: str, context: str, sio, agent_id: str, config, loop, hooks: Optional[dict] = None):
self.base_dir = base_dir self.base_dir = base_dir

71
Data/Server/WebUI/src/nodes/Agent Roles/Node_Agent_Role_Screenshot.jsx
View File

@@ -1,5 +1,5 @@
////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/WebUI/src/nodes/Agent/Node_Agent_Role_Screenshot.jsx ////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/WebUI/src/nodes/Agent/Node_Agent_Role_Screenshot.jsx
import React, { useEffect, useRef, useState } from "react"; import React, { useCallback, useEffect, useRef, useState } from "react";
import { Handle, Position, useReactFlow, useStore } from "reactflow"; import { Handle, Position, useReactFlow, useStore } from "reactflow";
import ShareIcon from "@mui/icons-material/Share"; import ShareIcon from "@mui/icons-material/Share";
import IconButton from "@mui/material/IconButton"; import IconButton from "@mui/material/IconButton";
@@ -21,6 +21,17 @@ const AgentScreenshotNode = ({ id, data }) => {
const { setNodes, getNodes } = useReactFlow(); const { setNodes, getNodes } = useReactFlow();
const edges = useStore(state => state.edges); const edges = useStore(state => state.edges);
const resolveAgentData = useCallback(() => {
try {
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner");
const agentNode = getNodes().find(n => n.id === agentEdge?.source);
return agentNode?.data || null;
} catch (err) {
return null;
}
}, [edges, getNodes, id]);
// Core config values pulled from sidebar config (with defaults) // Core config values pulled from sidebar config (with defaults)
const interval = parseInt(data?.interval || 1000, 10) || 1000; const interval = parseInt(data?.interval || 1000, 10) || 1000;
const region = { const region = {
@@ -32,6 +43,11 @@ const AgentScreenshotNode = ({ id, data }) => {
const visible = (data?.visible ?? "true") === "true"; const visible = (data?.visible ?? "true") === "true";
const alias = data?.alias || ""; const alias = data?.alias || "";
const [imageBase64, setImageBase64] = useState(data?.value || ""); const [imageBase64, setImageBase64] = useState(data?.value || "");
const agentData = resolveAgentData();
const targetModeLabel = ((agentData?.agent_mode || "").toString().toLowerCase() === "system")
? "SYSTEM Agent"
: "CURRENTUSER Agent";
const targetHostLabel = (agentData?.agent_host || "").toString();
// Always push current imageBase64 into BorealisValueBus at the global update rate // Always push current imageBase64 into BorealisValueBus at the global update rate
useEffect(() => { useEffect(() => {
@@ -56,14 +72,9 @@ const AgentScreenshotNode = ({ id, data }) => {
const handleScreenshot = (payload) => { const handleScreenshot = (payload) => {
if (payload?.node_id !== id) return; if (payload?.node_id !== id) return;
// Additionally ensure payload is from the agent connected upstream of this node // Additionally ensure payload is from the agent connected upstream of this node
try { const agentData = resolveAgentData();
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner"); const selectedAgentId = agentData?.agent_id;
const agentNode = getNodes().find(n => n.id === agentEdge?.source); if (!selectedAgentId || payload?.agent_id !== selectedAgentId) return;
const selectedAgentId = agentNode?.data?.agent_id;
if (!selectedAgentId || payload?.agent_id !== selectedAgentId) return;
} catch (err) {
return; // fail-closed if we cannot resolve upstream agent
}
if (payload.image_base64) { if (payload.image_base64) {
setImageBase64(payload.image_base64); setImageBase64(payload.image_base64);
@@ -86,24 +97,30 @@ const AgentScreenshotNode = ({ id, data }) => {
socket.on("agent_screenshot_task", handleScreenshot); socket.on("agent_screenshot_task", handleScreenshot);
return () => socket.off("agent_screenshot_task", handleScreenshot); return () => socket.off("agent_screenshot_task", handleScreenshot);
}, [id, setNodes, edges, getNodes]); }, [id, setNodes, resolveAgentData]);
// Register this node for the agent provisioning sync // Register this node for the agent provisioning sync
window.__BorealisInstructionNodes = window.__BorealisInstructionNodes || {}; window.__BorealisInstructionNodes = window.__BorealisInstructionNodes || {};
window.__BorealisInstructionNodes[id] = () => ({ window.__BorealisInstructionNodes[id] = () => {
node_id: id, const agentData = resolveAgentData() || {};
role: "screenshot", const modeRaw = (agentData.agent_mode || "").toString().toLowerCase();
interval, const targetMode = modeRaw === "system" ? "system" : "currentuser";
visible, return {
alias, node_id: id,
...region role: "screenshot",
}); interval,
visible,
alias,
target_agent_mode: targetMode,
target_agent_host: agentData.agent_host || "",
...region
};
};
// Manual live view copy button // Manual live view copy button
const handleCopyLiveViewLink = () => { const handleCopyLiveViewLink = () => {
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner"); const agentData = resolveAgentData();
const agentNode = getNodes().find(n => n.id === agentEdge?.source); const selectedAgentId = agentData?.agent_id;
const selectedAgentId = agentNode?.data?.agent_id;
if (!selectedAgentId) { if (!selectedAgentId) {
alert("No valid agent connection found."); alert("No valid agent connection found.");
@@ -132,6 +149,17 @@ const AgentScreenshotNode = ({ id, data }) => {
<div> <div>
<b>Interval:</b> {interval} ms <b>Interval:</b> {interval} ms
</div> </div>
<div>
<b>Agent Context:</b> {targetModeLabel}
</div>
<div>
<b>Target Host:</b>{" "}
{targetHostLabel ? (
targetHostLabel
) : (
<span style={{ color: "#666" }}>unknown</span>
)}
</div>
<div> <div>
<b>Overlay:</b> {visible ? "Yes" : "No"} <b>Overlay:</b> {visible ? "Yes" : "No"}
</div> </div>
@@ -165,6 +193,7 @@ Capture a live screenshot of a defined region from a remote Borealis Agent.
- Optionally show a visual overlay with a label - Optionally show a visual overlay with a label
- Pushes base64 PNG stream to downstream nodes - Pushes base64 PNG stream to downstream nodes
- Use copy button to share live view URL - Use copy button to share live view URL
- Targets the CURRENTUSER or SYSTEM agent context selected upstream
`.trim(), `.trim(),
content: "Capture screenshot region via agent", content: "Capture screenshot region via agent",
component: AgentScreenshotNode, component: AgentScreenshotNode,

198
Data/Server/WebUI/src/nodes/Agent/Node_Agent.jsx
View File

@@ -8,22 +8,59 @@ const BorealisAgentNode = ({ id, data }) => {
const edges = useStore((state) => state.edges); const edges = useStore((state) => state.edges);
const [agents, setAgents] = useState({}); const [agents, setAgents] = useState({});
const [selectedAgent, setSelectedAgent] = useState(data.agent_id || ""); const [selectedAgent, setSelectedAgent] = useState(data.agent_id || "");
const [selectedHost, setSelectedHost] = useState(data.agent_host || "");
const initialMode = (data.agent_mode || "currentuser").toLowerCase();
const [selectedMode, setSelectedMode] = useState(
initialMode === "system" ? "system" : "currentuser"
);
const [isConnected, setIsConnected] = useState(false); const [isConnected, setIsConnected] = useState(false);
const prevRolesRef = useRef([]); const prevRolesRef = useRef([]);
// Agent List Sorted (Online First) // Group agents by hostname and execution context
const agentList = useMemo(() => { const agentsByHostname = useMemo(() => {
if (!agents || typeof agents !== "object") return []; if (!agents || typeof agents !== "object") return {};
return Object.entries(agents) const grouped = {};
.map(([aid, info]) => ({ Object.entries(agents).forEach(([aid, info]) => {
id: aid, if (!info || typeof info !== "object") return;
status: info?.status || "offline", const status = (info.status || "").toString().toLowerCase();
last_seen: info?.last_seen || 0 if (status === "offline") return;
})) const host = (info.hostname || info.agent_hostname || "").trim() || "unknown";
.filter(({ status }) => status !== "offline") const modeRaw = (info.service_mode || "").toString().toLowerCase();
.sort((a, b) => b.last_seen - a.last_seen); const mode = modeRaw === "system" ? "system" : "currentuser";
if (!grouped[host]) {
grouped[host] = { currentuser: null, system: null };
}
grouped[host][mode] = {
agent_id: aid,
status: info.status || "offline",
last_seen: info.last_seen || 0,
info,
};
});
return grouped;
}, [agents]); }, [agents]);
const hostOptions = useMemo(() => {
const entries = Object.entries(agentsByHostname)
.map(([host, contexts]) => {
const candidates = [contexts.currentuser, contexts.system].filter(Boolean);
if (!candidates.length) return null;
const label = host;
const latest = Math.max(...candidates.map((r) => r.last_seen || 0));
return { host, label, contexts, latest };
})
.filter(Boolean)
.sort((a, b) => {
if (b.latest !== a.latest) return b.latest - a.latest;
return a.host.localeCompare(b.host);
});
return entries;
}, [agentsByHostname]);
// Fetch Agents Periodically // Fetch Agents Periodically
useEffect(() => { useEffect(() => {
const fetchAgents = () => { const fetchAgents = () => {
@@ -33,19 +70,83 @@ const BorealisAgentNode = ({ id, data }) => {
.catch(() => {}); .catch(() => {});
}; };
fetchAgents(); fetchAgents();
const interval = setInterval(fetchAgents, 4000); const interval = setInterval(fetchAgents, 10000); // Update Agent List Every 10 Seconds
return () => clearInterval(interval); return () => clearInterval(interval);
}, []); }, []);
// Ensure host selection stays aligned with available agents
useEffect(() => {
const hostExists = hostOptions.some((opt) => opt.host === selectedHost);
if (hostExists) return;
if (selectedAgent && agents[selectedAgent]) {
const info = agents[selectedAgent];
const inferredHost = (info?.hostname || info?.agent_hostname || "").trim() || "unknown";
if (inferredHost && inferredHost !== selectedHost) {
setSelectedHost(inferredHost);
return;
}
}
const fallbackHost = hostOptions[0]?.host || "";
if (fallbackHost !== selectedHost) {
setSelectedHost(fallbackHost);
}
if (!fallbackHost && selectedAgent) {
setSelectedAgent("");
}
}, [hostOptions, selectedHost, selectedAgent, agents]);
// Align agent selection with host/mode choice
useEffect(() => {
if (!selectedHost) {
if (selectedMode !== "currentuser") setSelectedMode("currentuser");
if (selectedAgent) setSelectedAgent("");
return;
}
const contexts = agentsByHostname[selectedHost];
if (!contexts) {
if (selectedMode !== "currentuser") setSelectedMode("currentuser");
if (selectedAgent) setSelectedAgent("");
return;
}
if (!contexts[selectedMode]) {
const fallbackMode = contexts.currentuser
? "currentuser"
: contexts.system
? "system"
: selectedMode;
if (fallbackMode !== selectedMode) {
setSelectedMode(fallbackMode);
return;
}
}
const activeContext = contexts[selectedMode];
const targetAgentId = activeContext?.agent_id || "";
if (targetAgentId !== selectedAgent) {
setSelectedAgent(targetAgentId);
}
}, [selectedHost, selectedMode, agentsByHostname, selectedAgent]);
// Sync node data with sidebar changes // Sync node data with sidebar changes
useEffect(() => { useEffect(() => {
setNodes((nds) => setNodes((nds) =>
nds.map((n) => nds.map((n) =>
n.id === id ? { ...n, data: { ...n.data, agent_id: selectedAgent } } : n n.id === id
? {
...n,
data: {
...n.data,
agent_id: selectedAgent,
agent_host: selectedHost,
agent_mode: selectedMode,
},
}
: n
) )
); );
setIsConnected(false); setIsConnected(false);
}, [selectedAgent, setNodes, id]); }, [selectedAgent, selectedHost, selectedMode, setNodes, id]);
// Attached Roles logic // Attached Roles logic
const attachedRoleIds = useMemo( const attachedRoleIds = useMemo(
@@ -109,11 +210,19 @@ const BorealisAgentNode = ({ id, data }) => {
// Status Label // Status Label
const selectedAgentStatus = useMemo(() => { const selectedAgentStatus = useMemo(() => {
if (!selectedAgent) return "Unassigned"; if (!selectedHost) return "Unassigned";
const agent = agents[selectedAgent]; const contexts = agentsByHostname[selectedHost];
if (!agent) return "Reconnecting..."; if (!contexts) return "Offline";
return agent.status === "provisioned" ? "Connected" : "Available"; const activeContext = contexts[selectedMode];
}, [agents, selectedAgent]); if (!selectedAgent || !activeContext) return "Unavailable";
const status = (activeContext.status || "").toString().toLowerCase();
if (status === "provisioned") return "Connected";
if (status === "orphaned") return "Available";
if (!status) return "Available";
return status.charAt(0).toUpperCase() + status.slice(1);
}, [agentsByHostname, selectedHost, selectedMode, selectedAgent]);
const activeHostContexts = selectedHost ? agentsByHostname[selectedHost] : null;
// Render (Sidebar handles config) // Render (Sidebar handles config)
return ( return (
@@ -126,22 +235,46 @@ const BorealisAgentNode = ({ id, data }) => {
style={{ top: "100%", background: "#58a6ff" }} style={{ top: "100%", background: "#58a6ff" }}
/> />
<div className="borealis-node-header">Borealis Agent</div> <div className="borealis-node-header">Device Agent</div>
<div className="borealis-node-content" style={{ fontSize: "9px" }}> <div className="borealis-node-content" style={{ fontSize: "9px" }}>
<label>Agent:</label> <label>Current Device:</label>
<select <select
value={selectedAgent} value={selectedHost}
onChange={(e) => setSelectedAgent(e.target.value)} onChange={(e) => setSelectedHost(e.target.value)}
style={{ width: "100%", marginBottom: "6px", fontSize: "9px" }} style={{ width: "100%", marginBottom: "6px", fontSize: "9px" }}
> >
<option value="">-- Select --</option> <option value="">-- Select --</option>
{agentList.map(({ id: aid, status }) => ( {hostOptions.map(({ host, label }) => (
<option key={aid} value={aid}> <option key={host} value={host}>
{aid} ({status}) {label}
</option> </option>
))} ))}
</select> </select>
<label>Available Agent Context(s):</label>
<select
value={selectedMode}
onChange={(e) => setSelectedMode(e.target.value)}
style={{ width: "100%", marginBottom: "2px", fontSize: "9px" }}
disabled={!selectedHost}
>
<option value="currentuser" disabled={!activeHostContexts?.currentuser}>
CURRENTUSER (Screen Capture / Macros)
</option>
<option value="system" disabled={!activeHostContexts?.system}>
SYSTEM (Scripts)
</option>
</select>
<div style={{ fontSize: "6px", color: "#aaa", marginBottom: "6px" }}>
Agent ID:{" "}
{selectedAgent ? (
<span style={{ color: "#eee" }}>{selectedAgent}</span>
) : (
<span style={{ color: "#666" }}>none</span>
)}
</div>
{isConnected ? ( {isConnected ? (
<button <button
onClick={handleDisconnect} onClick={handleDisconnect}
@@ -158,14 +291,6 @@ const BorealisAgentNode = ({ id, data }) => {
Connect to Agent Connect to Agent
</button> </button>
)} )}
<hr style={{ margin: "6px 0", borderColor: "#444" }} />
<div style={{ fontSize: "8px", color: "#aaa" }}>
Status: <strong>{selectedAgentStatus}</strong>
<br />
Attach <strong>Agent Role Nodes</strong> to define live behavior.
</div>
</div> </div>
</div> </div>
); );
@@ -174,12 +299,13 @@ const BorealisAgentNode = ({ id, data }) => {
// Node Registration Object with sidebar config and docs // Node Registration Object with sidebar config and docs
export default { export default {
type: "Borealis_Agent", type: "Borealis_Agent",
label: "Borealis Agent", label: "Device Agent",
description: ` description: `
Select and connect to a remote Borealis Agent. Select and connect to a remote Borealis Agent.
- Assign roles to agent dynamically by connecting "Agent Role" nodes. - Assign roles to agent dynamically by connecting "Agent Role" nodes.
- Auto-provisions agent as role assignments change. - Auto-provisions agent as role assignments change.
- See live agent status and re-connect/disconnect easily. - See live agent status and re-connect/disconnect easily.
- Choose between CURRENTUSER and SYSTEM contexts for each device.
`.trim(), `.trim(),
content: "Select and manage an Agent with dynamic roles", content: "Select and manage an Agent with dynamic roles",
component: BorealisAgentNode, component: BorealisAgentNode,
@@ -197,7 +323,7 @@ Select and connect to a remote Borealis Agent.
This node represents an available Borealis Agent (Python client) you can control from your workflow. This node represents an available Borealis Agent (Python client) you can control from your workflow.
#### Features #### Features
- **Select** an agent from the list of online agents. - **Select** a device and agent context (CURRENTUSER vs SYSTEM).
- **Connect/Disconnect** from the agent at any time. - **Connect/Disconnect** from the agent at any time.
- **Attach roles** (by connecting "Agent Role" nodes to this node's output handle) to assign behaviors dynamically. - **Attach roles** (by connecting "Agent Role" nodes to this node's output handle) to assign behaviors dynamically.
- **Live status** shows if the agent is available, connected, or offline. - **Live status** shows if the agent is available, connected, or offline.

83
Data/Server/server.py
View File

@@ -708,10 +708,12 @@ def _collect_agent_hash_records() -> List[Dict[str, Any]]:
try: try:
for agent_id, info in (registered_agents or {}).items(): for agent_id, info in (registered_agents or {}).items():
if agent_id and isinstance(agent_id, str) and agent_id.lower().endswith('-script'): mode = _normalize_service_mode(info.get('service_mode'), agent_id)
continue if mode != 'currentuser':
if info.get('is_script_agent'): if agent_id and isinstance(agent_id, str) and agent_id.lower().endswith('-script'):
continue continue
if info.get('is_script_agent'):
continue
_register( _register(
agent_id, agent_id,
info.get('agent_guid'), info.get('agent_guid'),
@@ -5846,20 +5848,29 @@ def get_agents():
# Collapse duplicates by hostname; prefer newer last_seen and non-script entries # Collapse duplicates by hostname; prefer newer last_seen and non-script entries
seen_by_hostname = {} seen_by_hostname = {}
for aid, info in (registered_agents or {}).items(): for aid, info in (registered_agents or {}).items():
# Hide script-execution agents from the public list
if aid and isinstance(aid, str) and aid.lower().endswith('-script'):
continue
if info.get('is_script_agent'):
continue
d = dict(info) d = dict(info)
mode = _normalize_service_mode(d.get('service_mode'), aid)
# Hide non-interactive script helper entries from the public list
if mode != 'currentuser':
if aid and isinstance(aid, str) and aid.lower().endswith('-script'):
continue
if info.get('is_script_agent'):
continue
d['service_mode'] = mode
ts = d.get('collector_active_ts') or 0 ts = d.get('collector_active_ts') or 0
d['collector_active'] = bool(ts and (now - float(ts) < 130)) d['collector_active'] = bool(ts and (now - float(ts) < 130))
host = (d.get('hostname') or '').strip() or 'unknown' host = (d.get('hostname') or '').strip() or 'unknown'
# Select best record per hostname: highest last_seen wins bucket = seen_by_hostname.setdefault(host, {})
cur = seen_by_hostname.get(host) cur = bucket.get(mode)
if not cur or int(d.get('last_seen') or 0) >= int(cur[1].get('last_seen') or 0): if not cur or int(d.get('last_seen') or 0) >= int(cur[1].get('last_seen') or 0):
seen_by_hostname[host] = (aid, d) bucket[mode] = (aid, d)
out = { aid: d for host, (aid, d) in seen_by_hostname.items() } out = {}
for host, bucket in seen_by_hostname.items():
for mode, (aid, d) in bucket.items():
d = dict(d)
d['hostname'] = (d.get('hostname') or '').strip() or host
d['service_mode'] = mode
out[aid] = d
return jsonify(out) return jsonify(out)
@@ -5869,6 +5880,28 @@ def get_agents():
## dayjs_to_ts removed; scheduling parsing now lives in job_scheduler ## dayjs_to_ts removed; scheduling parsing now lives in job_scheduler
def _normalize_service_mode(value, agent_id=None):
try:
if isinstance(value, str):
text = value.strip().lower()
else:
text = ''
except Exception:
text = ''
if not text and agent_id:
try:
aid = agent_id.lower()
if '-svc-' in aid or aid.endswith('-svc'):
return 'system'
except Exception:
pass
if text in {'system', 'svc', 'service', 'system_service'}:
return 'system'
if text in {'interactive', 'currentuser', 'user', 'current_user'}:
return 'currentuser'
return 'currentuser'
def _is_empty(v): def _is_empty(v):
return v is None or v == '' or v == [] or v == {} return v is None or v == '' or v == [] or v == {}
@@ -7252,12 +7285,15 @@ def handle_collector_status(data):
if not agent_id: if not agent_id:
return return
mode = _normalize_service_mode((data or {}).get('service_mode'), agent_id)
rec = registered_agents.setdefault(agent_id, {}) rec = registered_agents.setdefault(agent_id, {})
rec['agent_id'] = agent_id rec['agent_id'] = agent_id
if hostname: if hostname:
rec['hostname'] = hostname rec['hostname'] = hostname
if active: if active:
rec['collector_active_ts'] = time.time() rec['collector_active_ts'] = time.time()
if mode:
rec['service_mode'] = mode
# Helper: decide if a reported user string is a real interactive user # Helper: decide if a reported user string is a real interactive user
def _is_valid_interactive_user(s: str) -> bool: def _is_valid_interactive_user(s: str) -> bool:
@@ -7491,16 +7527,20 @@ def connect_agent(data):
except Exception: except Exception:
pass pass
service_mode = _normalize_service_mode((data or {}).get("service_mode"), agent_id)
rec = registered_agents.setdefault(agent_id, {}) rec = registered_agents.setdefault(agent_id, {})
rec["agent_id"] = agent_id rec["agent_id"] = agent_id
rec["hostname"] = rec.get("hostname", "unknown") rec["hostname"] = rec.get("hostname", "unknown")
rec["agent_operating_system"] = rec.get("agent_operating_system", "-") rec["agent_operating_system"] = rec.get("agent_operating_system", "-")
rec["last_seen"] = int(time.time()) rec["last_seen"] = int(time.time())
rec["status"] = "provisioned" if agent_id in agent_configurations else "orphaned" rec["status"] = "provisioned" if agent_id in agent_configurations else "orphaned"
# Flag script agents so they can be filtered out elsewhere if desired rec["service_mode"] = service_mode
# Flag non-interactive script agents so they can be filtered out elsewhere if desired
try: try:
if isinstance(agent_id, str) and agent_id.lower().endswith('-script'): if isinstance(agent_id, str) and agent_id.lower().endswith('-script'):
rec['is_script_agent'] = True rec['is_script_agent'] = service_mode != 'currentuser'
elif 'is_script_agent' in rec:
rec.pop('is_script_agent', None)
except Exception: except Exception:
pass pass
# If we already know the hostname for this agent, persist last_seen so it # If we already know the hostname for this agent, persist last_seen so it
@@ -7524,6 +7564,8 @@ def on_agent_heartbeat(data):
return return
hostname = data.get("hostname") hostname = data.get("hostname")
incoming_mode = _normalize_service_mode(data.get("service_mode"), agent_id)
if hostname: if hostname:
# Avoid duplicate entries per-hostname by collapsing to the newest agent_id. # Avoid duplicate entries per-hostname by collapsing to the newest agent_id.
# Prefer non-script agents; we do not surface script agents in /api/agents. # Prefer non-script agents; we do not surface script agents in /api/agents.
@@ -7537,6 +7579,9 @@ def on_agent_heartbeat(data):
if aid == agent_id: if aid == agent_id:
continue continue
if info.get("hostname") == hostname: if info.get("hostname") == hostname:
existing_mode = _normalize_service_mode(info.get("service_mode"), aid)
if existing_mode != incoming_mode:
continue
# If the incoming is a script helper and there is a non-script entry, keep non-script # If the incoming is a script helper and there is a non-script entry, keep non-script
if is_current_script and not info.get('is_script_agent'): if is_current_script and not info.get('is_script_agent'):
# Do not register duplicate script entry; just update last_seen persistence below # Do not register duplicate script entry; just update last_seen persistence below
@@ -7561,6 +7606,14 @@ def on_agent_heartbeat(data):
rec["agent_operating_system"] = data.get("agent_operating_system") rec["agent_operating_system"] = data.get("agent_operating_system")
rec["last_seen"] = int(data.get("last_seen") or time.time()) rec["last_seen"] = int(data.get("last_seen") or time.time())
rec["status"] = "provisioned" if agent_id in agent_configurations else rec.get("status", "orphaned") rec["status"] = "provisioned" if agent_id in agent_configurations else rec.get("status", "orphaned")
rec["service_mode"] = incoming_mode
try:
if isinstance(agent_id, str) and agent_id.lower().endswith('-script'):
rec['is_script_agent'] = incoming_mode != 'currentuser'
elif 'is_script_agent' in rec:
rec.pop('is_script_agent', None)
except Exception:
pass
# Persist last_seen (and agent_id) into DB keyed by hostname so it survives restarts. # Persist last_seen (and agent_id) into DB keyed by hostname so it survives restarts.
try: try:
_persist_last_seen(rec.get("hostname") or hostname, rec["last_seen"], rec.get("agent_id")) _persist_last_seen(rec.get("hostname") or hostname, rec["last_seen"], rec.get("agent_id"))