Support selecting agent context for screenshots

This commit is contained in:
2025-10-15 04:40:21 -06:00
parent a86231c8f5
commit 41c52590f4
6 changed files with 291 additions and 56 deletions

View File

@@ -87,6 +87,15 @@ def _coerce_text(value):
return ''
def _normalize_mode(value):
text = _coerce_text(value).strip().lower()
if text in {'interactive', 'currentuser', 'user'}:
return 'currentuser'
if text in {'system', 'svc', 'service'}:
return 'system'
return ''
class ScreenshotRegion(QtWidgets.QWidget):
def __init__(self, ctx, node_id, x=100, y=100, w=300, h=200, alias=None):
super().__init__()
@@ -376,6 +385,8 @@ class Role:
'h': _coerce_int(cfg.get('h'), 200, minimum=1),
'visible': _coerce_bool(cfg.get('visible'), True),
'alias': _coerce_text(cfg.get('alias')),
'target_agent_mode': _normalize_mode(cfg.get('target_agent_mode')),
'target_agent_host': _coerce_text(cfg.get('target_agent_host')),
}
return norm
except Exception:
@@ -387,6 +398,11 @@ class Role:
if not nid:
return
target_mode = cfg.get('target_agent_mode') or ''
current_mode = getattr(self.ctx, 'service_mode', '') or ''
if target_mode and current_mode and target_mode != current_mode:
return
alias = cfg.get('alias', '')
visible = cfg.get('visible', True)
reg = self.ctx.config.data.setdefault('regions', {})

View File

@@ -72,6 +72,7 @@ def _bootstrap_log(msg: str):
# Headless/service mode flag (skip Qt and interactive UI)
SYSTEM_SERVICE_MODE = ('--system-service' in sys.argv) or (os.environ.get('BOREALIS_AGENT_MODE') == 'system')
SERVICE_MODE = 'system' if SYSTEM_SERVICE_MODE else 'currentuser'
_bootstrap_log(f'agent.py loaded; SYSTEM_SERVICE_MODE={SYSTEM_SERVICE_MODE}; argv={sys.argv!r}')
def _argv_get(flag: str, default: str = None):
try:
@@ -859,7 +860,8 @@ async def send_heartbeat():
"agent_id": AGENT_ID,
"hostname": socket.gethostname(),
"agent_operating_system": detect_agent_os(),
"last_seen": int(time.time())
"last_seen": int(time.time()),
"service_mode": SERVICE_MODE,
}
await sio.emit("agent_heartbeat", payload)
# Also report collector status alive ping.
@@ -872,6 +874,7 @@ async def send_heartbeat():
'agent_id': AGENT_ID,
'hostname': socket.gethostname(),
'active': True,
'service_mode': SERVICE_MODE,
'last_user': f"{os.environ.get('USERDOMAIN') or socket.gethostname()}\\{getpass.getuser()}"
})
else:
@@ -879,6 +882,7 @@ async def send_heartbeat():
'agent_id': AGENT_ID,
'hostname': socket.gethostname(),
'active': True,
'service_mode': SERVICE_MODE,
})
except Exception:
pass
@@ -1203,7 +1207,7 @@ async def send_agent_details_once():
async def connect():
print(f"[INFO] Successfully Connected to Borealis Server!")
_log_agent('Connected to server.')
await sio.emit('connect_agent', {"agent_id": AGENT_ID})
await sio.emit('connect_agent', {"agent_id": AGENT_ID, "service_mode": SERVICE_MODE})
# Send an immediate heartbeat so the UI can populate instantly.
try:
@@ -1211,7 +1215,8 @@ async def connect():
"agent_id": AGENT_ID,
"hostname": socket.gethostname(),
"agent_operating_system": detect_agent_os(),
"last_seen": int(time.time())
"last_seen": int(time.time()),
"service_mode": SERVICE_MODE,
})
except Exception as e:
print(f"[WARN] initial heartbeat failed: {e}")
@@ -1225,6 +1230,7 @@ async def connect():
'agent_id': AGENT_ID,
'hostname': socket.gethostname(),
'active': True,
'service_mode': SERVICE_MODE,
'last_user': f"{os.environ.get('USERDOMAIN') or socket.gethostname()}\\{getpass.getuser()}"
})
else:
@@ -1232,6 +1238,7 @@ async def connect():
'agent_id': AGENT_ID,
'hostname': socket.gethostname(),
'active': True,
'service_mode': SERVICE_MODE,
})
except Exception:
pass
@@ -1542,9 +1549,10 @@ if __name__=='__main__':
# Initialize roles context for role tasks
# Initialize role manager and hot-load roles from Roles/
try:
hooks = {'send_service_control': send_service_control, 'get_server_url': get_server_url}
base_hooks = {'send_service_control': send_service_control, 'get_server_url': get_server_url}
if not SYSTEM_SERVICE_MODE:
# Load interactive-context roles (tray/UI, current-user execution, screenshot, etc.)
hooks_interactive = {**base_hooks, 'service_mode': 'currentuser'}
ROLE_MANAGER = RoleManager(
base_dir=os.path.dirname(__file__),
context='interactive',
@@ -1552,12 +1560,13 @@ if __name__=='__main__':
agent_id=AGENT_ID,
config=CONFIG,
loop=loop,
hooks=hooks,
hooks=hooks_interactive,
)
ROLE_MANAGER.load()
# Load system roles only when running in SYSTEM service mode
ROLE_MANAGER_SYS = None
if SYSTEM_SERVICE_MODE:
hooks_system = {**base_hooks, 'service_mode': 'system'}
ROLE_MANAGER_SYS = RoleManager(
base_dir=os.path.dirname(__file__),
context='system',
@@ -1565,7 +1574,7 @@ if __name__=='__main__':
agent_id=AGENT_ID,
config=CONFIG,
loop=loop,
hooks=hooks,
hooks=hooks_system,
)
ROLE_MANAGER_SYS.load()
except Exception as e:

View File

@@ -24,6 +24,10 @@ class RoleManager:
self.config = config
self.loop = loop
self.hooks = hooks or {}
try:
self.service_mode = (self.hooks.get('service_mode') or '').strip().lower()
except Exception:
self.service_mode = ''
def __init__(self, base_dir: str, context: str, sio, agent_id: str, config, loop, hooks: Optional[dict] = None):
self.base_dir = base_dir

View File

@@ -1,5 +1,5 @@
////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/WebUI/src/nodes/Agent/Node_Agent_Role_Screenshot.jsx
import React, { useEffect, useRef, useState } from "react";
import React, { useCallback, useEffect, useRef, useState } from "react";
import { Handle, Position, useReactFlow, useStore } from "reactflow";
import ShareIcon from "@mui/icons-material/Share";
import IconButton from "@mui/material/IconButton";
@@ -21,6 +21,17 @@ const AgentScreenshotNode = ({ id, data }) => {
const { setNodes, getNodes } = useReactFlow();
const edges = useStore(state => state.edges);
const resolveAgentData = useCallback(() => {
try {
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner");
const agentNode = getNodes().find(n => n.id === agentEdge?.source);
return agentNode?.data || null;
} catch (err) {
return null;
}
}, [edges, getNodes, id]);
// Core config values pulled from sidebar config (with defaults)
const interval = parseInt(data?.interval || 1000, 10) || 1000;
const region = {
@@ -32,6 +43,11 @@ const AgentScreenshotNode = ({ id, data }) => {
const visible = (data?.visible ?? "true") === "true";
const alias = data?.alias || "";
const [imageBase64, setImageBase64] = useState(data?.value || "");
const agentData = resolveAgentData();
const targetModeLabel = ((agentData?.agent_mode || "").toString().toLowerCase() === "system")
? "SYSTEM Agent"
: "CURRENTUSER Agent";
const targetHostLabel = (agentData?.agent_host || "").toString();
// Always push current imageBase64 into BorealisValueBus at the global update rate
useEffect(() => {
@@ -56,14 +72,9 @@ const AgentScreenshotNode = ({ id, data }) => {
const handleScreenshot = (payload) => {
if (payload?.node_id !== id) return;
// Additionally ensure payload is from the agent connected upstream of this node
try {
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner");
const agentNode = getNodes().find(n => n.id === agentEdge?.source);
const selectedAgentId = agentNode?.data?.agent_id;
if (!selectedAgentId || payload?.agent_id !== selectedAgentId) return;
} catch (err) {
return; // fail-closed if we cannot resolve upstream agent
}
const agentData = resolveAgentData();
const selectedAgentId = agentData?.agent_id;
if (!selectedAgentId || payload?.agent_id !== selectedAgentId) return;
if (payload.image_base64) {
setImageBase64(payload.image_base64);
@@ -86,24 +97,30 @@ const AgentScreenshotNode = ({ id, data }) => {
socket.on("agent_screenshot_task", handleScreenshot);
return () => socket.off("agent_screenshot_task", handleScreenshot);
}, [id, setNodes, edges, getNodes]);
}, [id, setNodes, resolveAgentData]);
// Register this node for the agent provisioning sync
window.__BorealisInstructionNodes = window.__BorealisInstructionNodes || {};
window.__BorealisInstructionNodes[id] = () => ({
node_id: id,
role: "screenshot",
interval,
visible,
alias,
...region
});
window.__BorealisInstructionNodes[id] = () => {
const agentData = resolveAgentData() || {};
const modeRaw = (agentData.agent_mode || "").toString().toLowerCase();
const targetMode = modeRaw === "system" ? "system" : "currentuser";
return {
node_id: id,
role: "screenshot",
interval,
visible,
alias,
target_agent_mode: targetMode,
target_agent_host: agentData.agent_host || "",
...region
};
};
// Manual live view copy button
const handleCopyLiveViewLink = () => {
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner");
const agentNode = getNodes().find(n => n.id === agentEdge?.source);
const selectedAgentId = agentNode?.data?.agent_id;
const agentData = resolveAgentData();
const selectedAgentId = agentData?.agent_id;
if (!selectedAgentId) {
alert("No valid agent connection found.");
@@ -132,6 +149,17 @@ const AgentScreenshotNode = ({ id, data }) => {
<div>
<b>Interval:</b> {interval} ms
</div>
<div>
<b>Agent Context:</b> {targetModeLabel}
</div>
<div>
<b>Target Host:</b>{" "}
{targetHostLabel ? (
targetHostLabel
) : (
<span style={{ color: "#666" }}>unknown</span>
)}
</div>
<div>
<b>Overlay:</b> {visible ? "Yes" : "No"}
</div>
@@ -165,6 +193,7 @@ Capture a live screenshot of a defined region from a remote Borealis Agent.
- Optionally show a visual overlay with a label
- Pushes base64 PNG stream to downstream nodes
- Use copy button to share live view URL
- Targets the CURRENTUSER or SYSTEM agent context selected upstream
`.trim(),
content: "Capture screenshot region via agent",
component: AgentScreenshotNode,

View File

@@ -8,22 +8,61 @@ const BorealisAgentNode = ({ id, data }) => {
const edges = useStore((state) => state.edges);
const [agents, setAgents] = useState({});
const [selectedAgent, setSelectedAgent] = useState(data.agent_id || "");
const [selectedHost, setSelectedHost] = useState(data.agent_host || "");
const initialMode = (data.agent_mode || "currentuser").toLowerCase();
const [selectedMode, setSelectedMode] = useState(
initialMode === "system" ? "system" : "currentuser"
);
const [isConnected, setIsConnected] = useState(false);
const prevRolesRef = useRef([]);
// Agent List Sorted (Online First)
const agentList = useMemo(() => {
if (!agents || typeof agents !== "object") return [];
return Object.entries(agents)
.map(([aid, info]) => ({
id: aid,
status: info?.status || "offline",
last_seen: info?.last_seen || 0
}))
.filter(({ status }) => status !== "offline")
.sort((a, b) => b.last_seen - a.last_seen);
// Group agents by hostname and execution context
const agentsByHostname = useMemo(() => {
if (!agents || typeof agents !== "object") return {};
const grouped = {};
Object.entries(agents).forEach(([aid, info]) => {
if (!info || typeof info !== "object") return;
const status = (info.status || "").toString().toLowerCase();
if (status === "offline") return;
const host = (info.hostname || info.agent_hostname || "").trim() || "unknown";
const modeRaw = (info.service_mode || "").toString().toLowerCase();
const mode = modeRaw === "system" ? "system" : "currentuser";
if (!grouped[host]) {
grouped[host] = { currentuser: null, system: null };
}
grouped[host][mode] = {
agent_id: aid,
status: info.status || "offline",
last_seen: info.last_seen || 0,
info,
};
});
return grouped;
}, [agents]);
const hostOptions = useMemo(() => {
const entries = Object.entries(agentsByHostname)
.map(([host, contexts]) => {
const candidates = [contexts.currentuser, contexts.system].filter(Boolean);
if (!candidates.length) return null;
const badge = (record) => {
if (!record) return "✕";
const st = (record.status || "").toString().toLowerCase();
if (st === "provisioned") return "✓";
return "•";
};
const label = `${host} (CURRENTUSER ${badge(contexts.currentuser)}, SYSTEM ${badge(contexts.system)})`;
const latest = Math.max(...candidates.map((r) => r.last_seen || 0));
return { host, label, contexts, latest };
})
.filter(Boolean)
.sort((a, b) => {
if (b.latest !== a.latest) return b.latest - a.latest;
return a.host.localeCompare(b.host);
});
return entries;
}, [agentsByHostname]);
// Fetch Agents Periodically
useEffect(() => {
const fetchAgents = () => {
@@ -37,15 +76,79 @@ const BorealisAgentNode = ({ id, data }) => {
return () => clearInterval(interval);
}, []);
// Ensure host selection stays aligned with available agents
useEffect(() => {
const hostExists = hostOptions.some((opt) => opt.host === selectedHost);
if (hostExists) return;
if (selectedAgent && agents[selectedAgent]) {
const info = agents[selectedAgent];
const inferredHost = (info?.hostname || info?.agent_hostname || "").trim() || "unknown";
if (inferredHost && inferredHost !== selectedHost) {
setSelectedHost(inferredHost);
return;
}
}
const fallbackHost = hostOptions[0]?.host || "";
if (fallbackHost !== selectedHost) {
setSelectedHost(fallbackHost);
}
if (!fallbackHost && selectedAgent) {
setSelectedAgent("");
}
}, [hostOptions, selectedHost, selectedAgent, agents]);
// Align agent selection with host/mode choice
useEffect(() => {
if (!selectedHost) {
if (selectedMode !== "currentuser") setSelectedMode("currentuser");
if (selectedAgent) setSelectedAgent("");
return;
}
const contexts = agentsByHostname[selectedHost];
if (!contexts) {
if (selectedMode !== "currentuser") setSelectedMode("currentuser");
if (selectedAgent) setSelectedAgent("");
return;
}
if (!contexts[selectedMode]) {
const fallbackMode = contexts.currentuser
? "currentuser"
: contexts.system
? "system"
: selectedMode;
if (fallbackMode !== selectedMode) {
setSelectedMode(fallbackMode);
return;
}
}
const activeContext = contexts[selectedMode];
const targetAgentId = activeContext?.agent_id || "";
if (targetAgentId !== selectedAgent) {
setSelectedAgent(targetAgentId);
}
}, [selectedHost, selectedMode, agentsByHostname, selectedAgent]);
// Sync node data with sidebar changes
useEffect(() => {
setNodes((nds) =>
nds.map((n) =>
n.id === id ? { ...n, data: { ...n.data, agent_id: selectedAgent } } : n
n.id === id
? {
...n,
data: {
...n.data,
agent_id: selectedAgent,
agent_host: selectedHost,
agent_mode: selectedMode,
},
}
: n
)
);
setIsConnected(false);
}, [selectedAgent, setNodes, id]);
}, [selectedAgent, selectedHost, selectedMode, setNodes, id]);
// Attached Roles logic
const attachedRoleIds = useMemo(
@@ -109,11 +212,19 @@ const BorealisAgentNode = ({ id, data }) => {
// Status Label
const selectedAgentStatus = useMemo(() => {
if (!selectedAgent) return "Unassigned";
const agent = agents[selectedAgent];
if (!agent) return "Reconnecting...";
return agent.status === "provisioned" ? "Connected" : "Available";
}, [agents, selectedAgent]);
if (!selectedHost) return "Unassigned";
const contexts = agentsByHostname[selectedHost];
if (!contexts) return "Offline";
const activeContext = contexts[selectedMode];
if (!selectedAgent || !activeContext) return "Unavailable";
const status = (activeContext.status || "").toString().toLowerCase();
if (status === "provisioned") return "Connected";
if (status === "orphaned") return "Available";
if (!status) return "Available";
return status.charAt(0).toUpperCase() + status.slice(1);
}, [agentsByHostname, selectedHost, selectedMode, selectedAgent]);
const activeHostContexts = selectedHost ? agentsByHostname[selectedHost] : null;
// Render (Sidebar handles config)
return (
@@ -128,20 +239,44 @@ const BorealisAgentNode = ({ id, data }) => {
<div className="borealis-node-header">Borealis Agent</div>
<div className="borealis-node-content" style={{ fontSize: "9px" }}>
<label>Agent:</label>
<label>Device:</label>
<select
value={selectedAgent}
onChange={(e) => setSelectedAgent(e.target.value)}
value={selectedHost}
onChange={(e) => setSelectedHost(e.target.value)}
style={{ width: "100%", marginBottom: "6px", fontSize: "9px" }}
>
<option value="">-- Select --</option>
{agentList.map(({ id: aid, status }) => (
<option key={aid} value={aid}>
{aid} ({status})
{hostOptions.map(({ host, label }) => (
<option key={host} value={host}>
{label}
</option>
))}
</select>
<label>Agent Context:</label>
<select
value={selectedMode}
onChange={(e) => setSelectedMode(e.target.value)}
style={{ width: "100%", marginBottom: "6px", fontSize: "9px" }}
disabled={!selectedHost}
>
<option value="currentuser" disabled={!activeHostContexts?.currentuser}>
CURRENTUSER Agent
</option>
<option value="system" disabled={!activeHostContexts?.system}>
SYSTEM Agent
</option>
</select>
<div style={{ fontSize: "8px", color: "#aaa", marginBottom: "4px" }}>
Target Agent ID:{" "}
{selectedAgent ? (
<span style={{ color: "#eee" }}>{selectedAgent}</span>
) : (
<span style={{ color: "#666" }}>none</span>
)}
</div>
{isConnected ? (
<button
onClick={handleDisconnect}
@@ -180,6 +315,7 @@ Select and connect to a remote Borealis Agent.
- Assign roles to agent dynamically by connecting "Agent Role" nodes.
- Auto-provisions agent as role assignments change.
- See live agent status and re-connect/disconnect easily.
- Choose between CURRENTUSER and SYSTEM contexts for each device.
`.trim(),
content: "Select and manage an Agent with dynamic roles",
component: BorealisAgentNode,
@@ -197,7 +333,7 @@ Select and connect to a remote Borealis Agent.
This node represents an available Borealis Agent (Python client) you can control from your workflow.
#### Features
- **Select** an agent from the list of online agents.
- **Select** a device and agent context (CURRENTUSER vs SYSTEM).
- **Connect/Disconnect** from the agent at any time.
- **Attach roles** (by connecting "Agent Role" nodes to this node's output handle) to assign behaviors dynamically.
- **Live status** shows if the agent is available, connected, or offline.

View File

@@ -5852,14 +5852,22 @@ def get_agents():
if info.get('is_script_agent'):
continue
d = dict(info)
mode = _normalize_service_mode(d.get('service_mode'), aid)
d['service_mode'] = mode
ts = d.get('collector_active_ts') or 0
d['collector_active'] = bool(ts and (now - float(ts) < 130))
host = (d.get('hostname') or '').strip() or 'unknown'
# Select best record per hostname: highest last_seen wins
cur = seen_by_hostname.get(host)
bucket = seen_by_hostname.setdefault(host, {})
cur = bucket.get(mode)
if not cur or int(d.get('last_seen') or 0) >= int(cur[1].get('last_seen') or 0):
seen_by_hostname[host] = (aid, d)
out = { aid: d for host, (aid, d) in seen_by_hostname.items() }
bucket[mode] = (aid, d)
out = {}
for host, bucket in seen_by_hostname.items():
for mode, (aid, d) in bucket.items():
d = dict(d)
d['hostname'] = (d.get('hostname') or '').strip() or host
d['service_mode'] = mode
out[aid] = d
return jsonify(out)
@@ -5869,6 +5877,28 @@ def get_agents():
## dayjs_to_ts removed; scheduling parsing now lives in job_scheduler
def _normalize_service_mode(value, agent_id=None):
try:
if isinstance(value, str):
text = value.strip().lower()
else:
text = ''
except Exception:
text = ''
if not text and agent_id:
try:
aid = agent_id.lower()
if '-svc-' in aid or aid.endswith('-svc'):
return 'system'
except Exception:
pass
if text in {'system', 'svc', 'service', 'system_service'}:
return 'system'
if text in {'interactive', 'currentuser', 'user', 'current_user'}:
return 'currentuser'
return 'currentuser'
def _is_empty(v):
return v is None or v == '' or v == [] or v == {}
@@ -7252,12 +7282,15 @@ def handle_collector_status(data):
if not agent_id:
return
mode = _normalize_service_mode((data or {}).get('service_mode'), agent_id)
rec = registered_agents.setdefault(agent_id, {})
rec['agent_id'] = agent_id
if hostname:
rec['hostname'] = hostname
if active:
rec['collector_active_ts'] = time.time()
if mode:
rec['service_mode'] = mode
# Helper: decide if a reported user string is a real interactive user
def _is_valid_interactive_user(s: str) -> bool:
@@ -7491,12 +7524,14 @@ def connect_agent(data):
except Exception:
pass
service_mode = _normalize_service_mode((data or {}).get("service_mode"), agent_id)
rec = registered_agents.setdefault(agent_id, {})
rec["agent_id"] = agent_id
rec["hostname"] = rec.get("hostname", "unknown")
rec["agent_operating_system"] = rec.get("agent_operating_system", "-")
rec["last_seen"] = int(time.time())
rec["status"] = "provisioned" if agent_id in agent_configurations else "orphaned"
rec["service_mode"] = service_mode
# Flag script agents so they can be filtered out elsewhere if desired
try:
if isinstance(agent_id, str) and agent_id.lower().endswith('-script'):
@@ -7524,6 +7559,8 @@ def on_agent_heartbeat(data):
return
hostname = data.get("hostname")
incoming_mode = _normalize_service_mode(data.get("service_mode"), agent_id)
if hostname:
# Avoid duplicate entries per-hostname by collapsing to the newest agent_id.
# Prefer non-script agents; we do not surface script agents in /api/agents.
@@ -7537,6 +7574,9 @@ def on_agent_heartbeat(data):
if aid == agent_id:
continue
if info.get("hostname") == hostname:
existing_mode = _normalize_service_mode(info.get("service_mode"), aid)
if existing_mode != incoming_mode:
continue
# If the incoming is a script helper and there is a non-script entry, keep non-script
if is_current_script and not info.get('is_script_agent'):
# Do not register duplicate script entry; just update last_seen persistence below
@@ -7561,6 +7601,7 @@ def on_agent_heartbeat(data):
rec["agent_operating_system"] = data.get("agent_operating_system")
rec["last_seen"] = int(data.get("last_seen") or time.time())
rec["status"] = "provisioned" if agent_id in agent_configurations else rec.get("status", "orphaned")
rec["service_mode"] = incoming_mode
# Persist last_seen (and agent_id) into DB keyed by hostname so it survives restarts.
try:
_persist_last_seen(rec.get("hostname") or hostname, rec["last_seen"], rec.get("agent_id"))