bunny-lab/Borealis
bunny-lab/Borealis
2
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Redesigned Agent Architecture

Browse Source
This commit is contained in:
Nicole Rappe 2025-04-15 20:27:23 -06:00
parent 17e0d20063
commit 8d67e847e9
9 changed files with 487 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
Data/Agent/borealis-agent.py
View File

@ -8,31 +8,28 @@ import threading
import socketio
from io import BytesIO
import socket
import os
from PyQt5 import QtCore, QtGui, QtWidgets
from PIL import ImageGrab
# ---------------- Configuration ----------------
SERVER_URL = "http://localhost:5000" # WebSocket-enabled Internal URL
#SERVER_URL = "https://borealis.bunny-lab.io" # WebSocket-enabled Public URL"
SERVER_URL = os.environ.get("SERVER_URL", "http://localhost:5000")
HOSTNAME = socket.gethostname().lower()
RANDOM_SUFFIX = uuid.uuid4().hex[:8]
AGENT_ID = f"{HOSTNAME}-agent-{RANDOM_SUFFIX}"
# ---------------- State ----------------
# ---------------- App State ----------------
app_instance = None
region_widget = None
current_interval = 1000
config_ready = threading.Event()
overlay_visible = True
overlay_widgets = {}
region_launchers = {}
running_roles = {}
running_threads = {}
LAST_CONFIG = {}
# WebSocket client setup
# ---------------- Socket Setup ----------------
sio = socketio.Client()
# ---------------- WebSocket Handlers ----------------
@sio.event
def connect():
print(f"[WebSocket] Agent ID: {AGENT_ID} connected to Borealis.")
@ -45,30 +42,18 @@ def disconnect():
@sio.on('agent_config')
def on_agent_config(config):
global current_interval, overlay_visible, LAST_CONFIG
print("[PROVISIONED] Received new configuration from Borealis.")
if config != LAST_CONFIG:
print("[PROVISIONED] Received new configuration from Borealis.")
x = config.get("x", 100)
y = config.get("y", 100)
w = config.get("w", 300)
h = config.get("h", 200)
current_interval = config.get("interval", 1000)
overlay_visible = config.get("visible", True)
roles = config.get("roles", [])
stop_all_roles()
for role in roles:
start_role_thread(role)
if not region_widget:
region_launcher.trigger.emit(x, y, w, h)
else:
region_widget.setGeometry(x, y, w, h)
region_widget.setVisible(overlay_visible)
LAST_CONFIG = config
config_ready.set()
# ---------------- Region Overlay ----------------
# ---------------- Overlay Class ----------------
class ScreenshotRegion(QtWidgets.QWidget):
def __init__(self, x=100, y=100, w=300, h=200):
def __init__(self, node_id, x=100, y=100, w=300, h=200):
super().__init__()
self.node_id = node_id
self.setGeometry(x, y, w, h)
self.setWindowFlags(QtCore.Qt.FramelessWindowHint | QtCore.Qt.WindowStaysOnTopHint)
self.setAttribute(QtCore.Qt.WA_TranslucentBackground)
@ -78,7 +63,7 @@ class ScreenshotRegion(QtWidgets.QWidget):
self.setVisible(True)
self.label = QtWidgets.QLabel(self)
self.label.setText(AGENT_ID)
self.label.setText(f"{node_id[:8]}")
self.label.setStyleSheet("color: lime; background: transparent; font-size: 10px;")
self.label.move(8, 4)
@ -87,7 +72,6 @@ class ScreenshotRegion(QtWidgets.QWidget):
def paintEvent(self, event):
painter = QtGui.QPainter(self)
painter.setRenderHint(QtGui.QPainter.Antialiasing)
painter.setBrush(QtCore.Qt.transparent)
painter.setPen(QtGui.QPen(QtGui.QColor(0, 255, 0), 2))
painter.drawRect(self.rect())
@ -102,8 +86,8 @@ class ScreenshotRegion(QtWidgets.QWidget):
def mousePressEvent(self, event):
if event.button() == QtCore.Qt.LeftButton:
if (event.pos().x() > self.width() - self.resize_handle_size and
event.pos().y() > self.height() - self.resize_handle_size):
if event.pos().x() > self.width() - self.resize_handle_size and \
event.pos().y() > self.height() - self.resize_handle_size:
self.resizing = True
else:
self.drag_offset = event.globalPos() - self.frameGeometry().topLeft()
@ -124,58 +108,91 @@ class ScreenshotRegion(QtWidgets.QWidget):
geo = self.geometry()
return geo.x(), geo.y(), geo.width(), geo.height()
# ---------------- Screenshot Capture ----------------
def capture_loop():
config_ready.wait()
while region_widget is None:
time.sleep(0.2)
while True:
if overlay_visible:
x, y, w, h = region_widget.get_geometry()
try:
img = ImageGrab.grab(bbox=(x, y, x + w, y + h))
buffer = BytesIO()
img.save(buffer, format="PNG")
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
sio.emit('screenshot', {
'agent_id': AGENT_ID,
'image_base64': encoded_image
})
except Exception as e:
print(f"[ERROR] Screenshot error: {e}")
time.sleep(current_interval / 1000)
# ---------------- UI Launcher ----------------
# ---------------- Region UI Handler ----------------
class RegionLauncher(QtCore.QObject):
trigger = QtCore.pyqtSignal(int, int, int, int)
def __init__(self):
def __init__(self, node_id):
super().__init__()
self.node_id = node_id
self.trigger.connect(self.handle)
def handle(self, x, y, w, h):
launch_region(x, y, w, h)
print(f"[Overlay] Launching overlay for {self.node_id} at ({x},{y},{w},{h})")
if self.node_id in overlay_widgets:
return
widget = ScreenshotRegion(self.node_id, x, y, w, h)
overlay_widgets[self.node_id] = widget
widget.show()
region_launcher = None
# ---------------- Role Management ----------------
def stop_all_roles():
for node_id, thread in running_threads.items():
if thread and thread.is_alive():
print(f"[Role] Terminating previous task: {node_id}")
running_roles.clear()
running_threads.clear()
def launch_region(x, y, w, h):
global region_widget
if region_widget:
def start_role_thread(role_cfg):
role = role_cfg.get("role")
node_id = role_cfg.get("node_id")
if not role or not node_id:
print("[ERROR] Invalid role configuration (missing role or node_id).")
return
region_widget = ScreenshotRegion(x, y, w, h)
region_widget.show()
if role == "screenshot":
thread = threading.Thread(target=run_screenshot_loop, args=(node_id, role_cfg), daemon=True)
else:
print(f"[SKIP] Unknown role: {role}")
return
running_roles[node_id] = role_cfg
running_threads[node_id] = thread
thread.start()
print(f"[Role] Started task: {role} ({node_id})")
# ---------------- Screenshot Role Loop ----------------
def run_screenshot_loop(node_id, cfg):
interval = cfg.get("interval", 1000)
visible = cfg.get("visible", True)
x = cfg.get("x", 100)
y = cfg.get("y", 100)
w = cfg.get("w", 300)
h = cfg.get("h", 200)
if node_id not in region_launchers:
launcher = RegionLauncher(node_id)
region_launchers[node_id] = launcher
launcher.trigger.emit(x, y, w, h)
widget = overlay_widgets.get(node_id)
if widget:
widget.setGeometry(x, y, w, h)
widget.setVisible(visible)
while True:
try:
if node_id in overlay_widgets:
widget = overlay_widgets[node_id]
x, y, w, h = widget.get_geometry()
print(f"[Capture] Screenshot task {node_id} at ({x},{y},{w},{h})")
img = ImageGrab.grab(bbox=(x, y, x + w, y + h))
buffer = BytesIO()
img.save(buffer, format="PNG")
encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
sio.emit("agent_screenshot_task", {
"agent_id": AGENT_ID,
"node_id": node_id,
"image_base64": encoded
})
except Exception as e:
print(f"[ERROR] Screenshot task {node_id} failed: {e}")
time.sleep(interval / 1000)
# ---------------- Main ----------------
if __name__ == "__main__":
app_instance = QtWidgets.QApplication(sys.argv)
region_launcher = RegionLauncher()
sio.connect(SERVER_URL, transports=['websocket'])
threading.Thread(target=capture_loop, daemon=True).start()
sio.connect(SERVER_URL, transports=["websocket"])
sys.exit(app_instance.exec_())

9
Data/WebUI/src/App.jsx
View File

@ -48,6 +48,15 @@ import React, {
} from "./Dialogs";
import StatusBar from "./Status_Bar";
// Websocket Functionality
import { io } from "socket.io-client";
if (!window.BorealisSocket) {
window.BorealisSocket = io(window.location.origin, {
transports: ["websocket"]
});
}
// Global Node Update Timer Variable
if (!window.BorealisUpdateRate) {
window.BorealisUpdateRate = 200;

2
Data/WebUI/src/Node_Sidebar.jsx
View File

@ -35,7 +35,7 @@ export default function NodeSidebar({
return (
<div
style={{
width: 320,
width: 300, //Width of the Node Sidebar
backgroundColor: "#121212",
borderRight: "1px solid #333",
overflowY: "auto"

154
Data/WebUI/src/nodes/Agents/Node_Agent.jsx Normal file
View File

@ -0,0 +1,154 @@
////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/WebUI/src/nodes/Agent/Node_Agent.jsx
import React, { useEffect, useState } from "react";
import { Handle, Position, useReactFlow, useStore } from "reactflow";
const BorealisAgentNode = ({ id, data }) => {
const { getNodes, getEdges, setNodes } = useReactFlow();
const [agents, setAgents] = useState([]);
const [selectedAgent, setSelectedAgent] = useState(data.agent_id || "");
// -------------------------------
// Load agent list from backend
// -------------------------------
useEffect(() => {
fetch("/api/agents")
.then(res => res.json())
.then(setAgents);
const interval = setInterval(() => {
fetch("/api/agents")
.then(res => res.json())
.then(setAgents);
}, 5000);
return () => clearInterval(interval);
}, []);
// -------------------------------
// Helper: Get all provisioner role nodes connected to bottom port
// -------------------------------
const getAttachedProvisioners = () => {
const allNodes = getNodes();
const allEdges = getEdges();
const attached = [];
for (const edge of allEdges) {
if (edge.source === id && edge.sourceHandle === "provisioner") {
const roleNode = allNodes.find(n => n.id === edge.target);
if (roleNode && typeof window.__BorealisInstructionNodes?.[roleNode.id] === "function") {
attached.push(window.__BorealisInstructionNodes[roleNode.id]());
}
}
}
return attached;
};
// -------------------------------
// Provision Agent with all Roles
// -------------------------------
const handleProvision = () => {
if (!selectedAgent) return;
const provisionRoles = getAttachedProvisioners();
if (!provisionRoles.length) {
console.warn("No provisioner nodes connected to agent.");
return;
}
const configPayload = {
agent_id: selectedAgent,
roles: provisionRoles
};
fetch("/api/agent/provision", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(configPayload)
})
.then(res => res.json())
.then(() => {
console.log(`[Provision] Agent ${selectedAgent} updated with ${provisionRoles.length} roles.`);
});
};
return (
<div className="borealis-node">
{/* This bottom port is used for bi-directional provisioning & feedback */}
<Handle
type="source"
position={Position.Bottom}
id="provisioner"
className="borealis-handle"
style={{ top: "100%", background: "#58a6ff" }}
/>
<div className="borealis-node-header">Borealis Agent</div>
<div className="borealis-node-content" style={{ fontSize: "9px" }}>
<label>Agent:</label>
<select
value={selectedAgent}
onChange={(e) => {
const newId = e.target.value;
setSelectedAgent(newId);
setNodes(nds =>
nds.map(n =>
n.id === id
? { ...n, data: { ...n.data, agent_id: newId } }
: n
)
);
}}
style={{ width: "100%", marginBottom: "6px", fontSize: "9px" }}
>
<option value="">-- Select --</option>
{Object.entries(agents).map(([id, info]) => {
const label = info.status === "provisioned" ? "(Provisioned)" : "(Idle)";
return (
<option key={id} value={id}>
{id} {label}
</option>
);
})}
</select>
<button
onClick={handleProvision}
style={{ width: "100%", fontSize: "9px", padding: "4px", marginTop: "4px" }}
>
Provision Agent
</button>
<hr style={{ margin: "6px 0", borderColor: "#444" }} />
<div style={{ fontSize: "8px", color: "#aaa" }}>
Connect <strong>Instruction Nodes</strong> below to define roles.
Each instruction node will send back its results (like screenshots) and act as a separate data output.
</div>
<div style={{ fontSize: "8px", color: "#aaa", marginTop: "4px" }}>
<strong>Supported Roles:</strong>
<ul style={{ paddingLeft: "14px", marginTop: "2px", marginBottom: "0" }}>
<li><code>screenshot</code>: Capture a region with interval and overlay</li>
{/* Future roles will be listed here */}
</ul>
</div>
</div>
</div>
);
};
export default {
type: "Borealis_Agent",
label: "Borealis Agent",
description: `
Main Agent Node
- Selects an available agent
- Connect instruction nodes below to assign tasks (roles)
- Roles include screenshots, keyboard macros, etc.
`.trim(),
content: "Select and provision a Borealis Agent with task roles",
component: BorealisAgentNode
};

186
Data/WebUI/src/nodes/Agents/Node_Agent_Role_Screenshot.jsx Normal file
View File

@ -0,0 +1,186 @@
////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/WebUI/src/nodes/Agent/Node_Agent_Role_Screenshot.jsx
import React, { useEffect, useRef, useState } from "react";
import { Handle, Position, useReactFlow, useStore } from "reactflow";
import ShareIcon from "@mui/icons-material/Share";
import IconButton from "@mui/material/IconButton";
if (!window.BorealisValueBus) {
window.BorealisValueBus = {};
}
if (!window.BorealisUpdateRate) {
window.BorealisUpdateRate = 100;
}
const ScreenshotInstructionNode = ({ id, data }) => {
const { setNodes, getNodes } = useReactFlow();
const edges = useStore(state => state.edges);
const [interval, setInterval] = useState(data?.interval || 1000);
const [region, setRegion] = useState({
x: data?.x ?? 250,
y: data?.y ?? 100,
w: data?.w ?? 300,
h: data?.h ?? 200,
});
const [visible, setVisible] = useState(data?.visible ?? true);
const [alias, setAlias] = useState(data?.alias || "");
const [imageBase64, setImageBase64] = useState("");
const base64Ref = useRef("");
const handleCopyLiveViewLink = () => {
const agentEdge = edges.find(e => e.target === id && e.sourceHandle === "provisioner");
if (!agentEdge) {
alert("No upstream agent connection found.");
return;
}
const agentNode = getNodes().find(n => n.id === agentEdge.source);
const selectedAgentId = agentNode?.data?.agent_id;
if (!selectedAgentId) {
alert("Upstream agent node does not have a selected agent.");
return;
}
const liveUrl = `${window.location.origin}/api/agent/${selectedAgentId}/node/${id}/screenshot/live`;
navigator.clipboard.writeText(liveUrl)
.then(() => console.log(`[Clipboard] Copied Live View URL: ${liveUrl}`))
.catch(err => console.error("Clipboard copy failed:", err));
};
useEffect(() => {
const intervalId = setInterval(() => {
const val = base64Ref.current;
console.log(`[Screenshot Node] setInterval update. Current base64 length: ${val?.length || 0}`);
if (!val) return;
window.BorealisValueBus[id] = val;
setNodes(nds =>
nds.map(n =>
n.id === id
? { ...n, data: { ...n.data, value: val } }
: n
)
);
}, window.BorealisUpdateRate || 100);
return () => clearInterval(intervalId);
}, [id, setNodes]);
useEffect(() => {
const socket = window.BorealisSocket || null;
if (!socket) {
console.warn("[Screenshot Node] BorealisSocket not available");
return;
}
console.log(`[Screenshot Node] Listening for agent_screenshot_task with node_id: ${id}`);
const handleScreenshot = (payload) => {
console.log("[Screenshot Node] Received payload:", payload);
if (payload?.node_id === id && payload?.image_base64) {
base64Ref.current = payload.image_base64;
setImageBase64(payload.image_base64);
window.BorealisValueBus[id] = payload.image_base64;
console.log(`[Screenshot Node] Updated base64Ref and ValueBus for ${id}, length: ${payload.image_base64.length}`);
} else {
console.log(`[Screenshot Node] Ignored payload for mismatched node_id (${payload?.node_id})`);
}
};
socket.on("agent_screenshot_task", handleScreenshot);
return () => socket.off("agent_screenshot_task", handleScreenshot);
}, [id]);
window.__BorealisInstructionNodes = window.__BorealisInstructionNodes || {};
window.__BorealisInstructionNodes[id] = () => ({
node_id: id,
role: "screenshot",
interval,
visible,
alias,
...region
});
return (
<div className="borealis-node" style={{ position: "relative" }}>
<Handle type="target" position={Position.Left} className="borealis-handle" />
<Handle type="source" position={Position.Right} className="borealis-handle" />
<div className="borealis-node-header">Agent Role: Screenshot</div>
<div className="borealis-node-content" style={{ fontSize: "9px" }}>
<label>Update Interval (ms):</label>
<input
type="number"
min="100"
step="100"
value={interval}
onChange={(e) => setInterval(Number(e.target.value))}
style={{ width: "100%", marginBottom: "4px" }}
/>
<label>Region X / Y / W / H:</label>
<div style={{ display: "flex", gap: "4px", marginBottom: "4px" }}>
<input type="number" value={region.x} onChange={(e) => setRegion({ ...region, x: Number(e.target.value) })} style={{ width: "25%" }} />
<input type="number" value={region.y} onChange={(e) => setRegion({ ...region, y: Number(e.target.value) })} style={{ width: "25%" }} />
<input type="number" value={region.w} onChange={(e) => setRegion({ ...region, w: Number(e.target.value) })} style={{ width: "25%" }} />
<input type="number" value={region.h} onChange={(e) => setRegion({ ...region, h: Number(e.target.value) })} style={{ width: "25%" }} />
</div>
<div style={{ marginBottom: "4px" }}>
<label>
<input
type="checkbox"
checked={visible}
onChange={() => setVisible(!visible)}
style={{ marginRight: "4px" }}
/>
Show Overlay on Agent
</label>
</div>
<label>Overlay Label:</label>
<input
type="text"
value={alias}
onChange={(e) => setAlias(e.target.value)}
placeholder="Label (optional)"
style={{ width: "100%", fontSize: "9px", marginBottom: "6px" }}
/>
<div style={{ textAlign: "center", fontSize: "8px", color: "#aaa" }}>
{imageBase64
? `Last image: ${Math.round(imageBase64.length / 1024)} KB`
: "Awaiting Screenshot Data..."}
</div>
</div>
<div style={{ position: "absolute", top: 4, right: 4 }}>
<IconButton size="small" onClick={handleCopyLiveViewLink}>
<ShareIcon style={{ fontSize: 14 }} />
</IconButton>
</div>
</div>
);
};
export default {
type: "Agent_Role_Screenshot",
label: "Agent Role: Screenshot",
description: `
Agent Role Node: Screenshot Region
- Defines a single region capture role
- Allows custom update interval and overlay
- Emits captured base64 PNG data from agent
`.trim(),
content: "Capture screenshot region via agent",
component: ScreenshotInstructionNode
};

2
Data/WebUI/src/nodes/Data Analysis/Node_OCR_Text_Extraction.jsx
View File

@ -233,7 +233,7 @@ const numberInputStyle = {
export default {
type: "OCR_Text_Extraction",
label: "OCR-Based Text Extraction",
label: "OCR Text Extraction",
description: `
Extract text from upstream image using backend OCR engine via API.
Includes rate limiting and sensitivity detection for smart processing.`,

172
Data/WebUI/src/nodes/Data Collection/Node_Borealis_Agent.jsx
View File

@ -1,172 +0,0 @@
////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: Node_Borealis_Agent.jsx
import React, { useEffect, useState, useRef } from "react";
import { Handle, Position, useReactFlow } from "reactflow";
import { io } from "socket.io-client";
const socket = io(window.location.origin, {
transports: ["websocket"]
});
const BorealisAgentNode = ({ id, data }) => {
const { setNodes } = useReactFlow();
const [agents, setAgents] = useState([]);
const [selectedAgent, setSelectedAgent] = useState(data.agent_id || "");
const [selectedType, setSelectedType] = useState(data.data_type || "screenshot");
const [intervalMs, setIntervalMs] = useState(data.interval || 1000);
const [paused, setPaused] = useState(false);
const [overlayVisible, setOverlayVisible] = useState(true);
const [imageData, setImageData] = useState("");
const imageRef = useRef("");
useEffect(() => {
fetch("/api/agents").then(res => res.json()).then(setAgents);
const interval = setInterval(() => {
fetch("/api/agents").then(res => res.json()).then(setAgents);
}, 5000);
return () => clearInterval(interval);
}, []);
useEffect(() => {
socket.on('new_screenshot', (data) => {
if (data.agent_id === selectedAgent) {
setImageData(data.image_base64);
imageRef.current = data.image_base64;
}
});
return () => socket.off('new_screenshot');
}, [selectedAgent]);
useEffect(() => {
const interval = setInterval(() => {
if (!paused && imageRef.current) {
window.BorealisValueBus = window.BorealisValueBus || {};
window.BorealisValueBus[id] = imageRef.current;
setNodes(nds => {
const updated = [...nds];
const node = updated.find(n => n.id === id);
if (node) {
node.data = {
...node.data,
value: imageRef.current
};
}
return updated;
});
}
}, window.BorealisUpdateRate || 100);
return () => clearInterval(interval);
}, [id, paused, setNodes]);
const provisionAgent = () => {
if (!selectedAgent) return;
fetch("/api/agent/provision", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
agent_id: selectedAgent,
x: 250,
y: 100,
w: 300,
h: 200,
interval: intervalMs,
visible: overlayVisible,
task: selectedType
})
})
.then(res => res.json())
.then(() => {
console.log("[DEBUG] Agent provisioned");
});
};
const toggleOverlay = () => {
const newVisibility = !overlayVisible;
setOverlayVisible(newVisibility);
provisionAgent();
};
return (
<div className="borealis-node">
<Handle type="source" position={Position.Right} className="borealis-handle" />
<div className="borealis-node-header">Borealis Agent</div>
<div className="borealis-node-content">
<label style={{ fontSize: "10px" }}>Agent:</label>
<select
value={selectedAgent}
onChange={(e) => setSelectedAgent(e.target.value)}
style={{ width: "100%", fontSize: "9px", marginBottom: "6px" }}
>
<option value="">-- Select --</option>
{Object.entries(agents).map(([id, info]) => {
const statusLabel = info.status === "provisioned"
? "(Provisioned)"
: "(Not Provisioned)";
return (
<option key={id} value={id}>
{id} {statusLabel}
</option>
);
})}
</select>
<label style={{ fontSize: "10px" }}>Data Type:</label>
<select
value={selectedType}
onChange={(e) => setSelectedType(e.target.value)}
style={{ width: "100%", fontSize: "9px", marginBottom: "6px" }}
>
<option value="screenshot">Screenshot Region</option>
</select>
<label style={{ fontSize: "10px" }}>Update Rate (ms):</label>
<input
type="number"
min="100"
step="100"
value={intervalMs}
onChange={(e) => setIntervalMs(Number(e.target.value))}
style={{ width: "100%", fontSize: "9px", marginBottom: "6px" }}
/>
<div style={{ marginBottom: "6px" }}>
<label style={{ fontSize: "10px" }}>
<input
type="checkbox"
checked={paused}
onChange={() => setPaused(!paused)}
style={{ marginRight: "4px" }}
/>
Pause Data Collection
</label>
</div>
<div style={{ display: "flex", gap: "4px", flexWrap: "wrap" }}>
<button
style={{ flex: 1, fontSize: "9px" }}
onClick={provisionAgent}
>
(Re)Provision
</button>
<button
style={{ flex: 1, fontSize: "9px" }}
onClick={toggleOverlay}
>
{overlayVisible ? "Hide Overlay" : "Show Overlay"}
</button>
</div>
</div>
</div>
);
};
export default {
type: "Borealis_Agent",
label: "Borealis Agent",
description: "Connects to and controls a Borealis Agent via WebSocket in real-time.",
content: "Provisions a Borealis Agent and streams collected data into the workflow graph.",
component: BorealisAgentNode
};

127
Data/server.py
View File

@ -73,38 +73,36 @@ def get_agents():
def provision_agent():
data = request.json
agent_id = data.get("agent_id")
config = {
"task": data.get("task", "screenshot"),
"x": data.get("x", 100),
"y": data.get("y", 100),
"w": data.get("w", 300),
"h": data.get("h", 200),
"interval": data.get("interval", 1000),
"visible": data.get("visible", True)
}
roles = data.get("roles", []) # <- MODULAR ROLES ARRAY
if not agent_id or not isinstance(roles, list):
return jsonify({"error": "Missing agent_id or roles[] in provision payload."}), 400
# Save configuration
config = {"roles": roles}
agent_configurations[agent_id] = config
# Update status if agent already registered
if agent_id in registered_agents:
registered_agents[agent_id]["status"] = "provisioned"
# NEW: Emit config update back to the agent via WebSocket
# Emit config to the agent
socketio.emit("agent_config", config)
return jsonify({"status": "provisioned"})
return jsonify({"status": "provisioned", "roles": roles})
# ----------------------------------------------
# Canvas Image Feed Viewer for Screenshot Agents
# ----------------------------------------------
@app.route("/api/agent/<agent_id>/screenshot/live")
def screenshot_viewer(agent_id):
if agent_configurations.get(agent_id, {}).get("task") != "screenshot":
return "<h1>Agent not provisioned as Screenshot Collector</h1>", 400
@app.route("/api/agent/<agent_id>/node/<node_id>/screenshot/live")
def screenshot_node_viewer(agent_id, node_id):
return f"""
<!DOCTYPE html>
<html>
<head>
<title>Borealis Live View - {agent_id}</title>
<title>Borealis Live View - {agent_id}:{node_id}</title>
<style>
body {{
margin: 0;
@ -118,105 +116,66 @@ def screenshot_viewer(agent_id):
border: 1px solid #444;
max-width: 90vw;
max-height: 90vh;
width: auto;
height: auto;
background-color: #111;
}}
</style>
</head>
<body>
<canvas id="viewerCanvas"></canvas>
<script src="https://cdn.socket.io/4.8.1/socket.io.min.js"></script>
<script>
const agentId = "{agent_id}";
const socket = io(window.location.origin, {{ transports: ["websocket"] }});
const nodeId = "{node_id}";
const canvas = document.getElementById("viewerCanvas");
const ctx = canvas.getContext("2d");
const socket = io(window.location.origin, {{ transports: ["websocket"] }});
console.log("[Viewer] Canvas initialized for agent:", agentId);
socket.on("connect", () => {{
console.log("[WebSocket] Connected to Borealis server at", window.location.origin);
}});
socket.on("disconnect", () => {{
console.warn("[WebSocket] Disconnected from Borealis server");
}});
socket.on("new_screenshot", (data) => {{
console.log("[WebSocket] Received screenshot event");
if (!data || typeof data !== "object") {{
console.error("[Viewer] Screenshot event was not an object:", data);
return;
}}
if (data.agent_id !== agentId) {{
console.log("[Viewer] Ignored screenshot from different agent:", data.agent_id);
return;
}}
socket.on("agent_screenshot_task", (data) => {{
if (data.agent_id !== agentId || data.node_id !== nodeId) return;
const base64 = data.image_base64;
console.log("[Viewer] Base64 length:", base64?.length || 0);
if (!base64 || base64.length < 100) {{
console.warn("[Viewer] Empty or too short base64 string.");
return;
}}
// Peek at base64 to determine MIME type
let mimeType = "image/png";
try {{
const header = atob(base64.substring(0, 32));
if (header.charCodeAt(0) === 0xFF && header.charCodeAt(1) === 0xD8) {{
mimeType = "image/jpeg";
}}
}} catch (e) {{
console.warn("[Viewer] Failed to decode base64 header", e);
}}
if (!base64 || base64.length < 100) return;
const img = new Image();
img.onload = () => {{
console.log("[Viewer] Image loaded successfully:", img.width + "x" + img.height);
console.log("[Viewer] Canvas size before:", canvas.width + "x" + canvas.height);
if (canvas.width !== img.width || canvas.height !== img.height) {{
canvas.width = img.width;
canvas.height = img.height;
console.log("[Viewer] Canvas resized to match image");
}}
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(img, 0, 0);
console.log("[Viewer] Image drawn on canvas");
}};
img.onerror = (err) => {{
console.error("[Viewer] Failed to load image from base64. Possibly corrupted data?", err);
}};
img.src = "data:" + mimeType + ";base64," + base64;
img.src = "data:image/png;base64," + base64;
}});
</script>
</body>
</html>
"""
@app.route("/api/agent/<agent_id>/screenshot/raw") # Fallback Non-Live Screenshot Preview Code for Legacy Purposes
def screenshot_raw(agent_id):
entry = latest_images.get(agent_id)
if not entry:
return "", 204
try:
raw_img = base64.b64decode(entry["image_base64"])
return Response(raw_img, mimetype="image/png")
except Exception:
return "", 204
# ---------------------------------------------
# WebSocket Events
# ---------------------------------------------
@socketio.on("agent_screenshot_task")
def receive_screenshot_task(data):
agent_id = data.get("agent_id")
node_id = data.get("node_id")
image = data.get("image_base64")
if not agent_id or not node_id or not image:
print("[WS] Screenshot task missing fields.")
return
# Optional: Store for debugging
latest_images[f"{agent_id}:{node_id}"] = {
"image_base64": image,
"timestamp": time.time()
}
emit("agent_screenshot_task", {
"agent_id": agent_id,
"node_id": node_id,
"image_base64": image
}, broadcast=True)
@socketio.on('connect_agent')
def connect_agent(data):
agent_id = data.get("agent_id")

2
Launch-Borealis.ps1
View File

@ -102,7 +102,7 @@ switch ($choice) {
}
# React UI Deployment: Create default React app if no deployment folder exists
if (-not (Test-Path $webUIDestination)) {
npx --yes create-react-app "$webUIDestination" --verbose | Out-Null
npx --yes create-react-app "$webUIDestination" | Out-Null
}
# Copy custom UI if it exists
if (Test-Path $customUIPath) {