Agent Multi-Role Milestone

Data/Agent/borealis-agent.py

@@ -21,9 +21,11 @@ from PIL import ImageGrab
 # //////////////////////////////////////////////////////////////////////////
 CONFIG_PATH = os.path.join(os.path.dirname(__file__), "agent_settings.json")
 DEFAULT_CONFIG = {
-    "SERVER_URL": "http://localhost:5000",
+    "borealis_server_url": "http://localhost:5000",
-    "max_workers": 8,
+    "max_task_workers": 8,
-    "config_watch_interval": 2
+    "config_file_watcher_interval": 2,
+    "agent_id": "",
+    "regions": {}
 }
 
 class ConfigManager:
@@ -34,16 +36,20 @@ class ConfigManager:
         self.load()
 
     def load(self):
+        # load or initialize
         if not os.path.exists(self.path):
             self.data = DEFAULT_CONFIG.copy()
             self._write()
         else:
             try:
                 with open(self.path, 'r') as f:
-                    self.data = json.load(f)
+                    loaded = json.load(f)
+                # merge defaults
+                self.data = {**DEFAULT_CONFIG, **loaded}
             except Exception as e:
                 print(f"[WARN] Failed to parse config: {e}")
                 self.data = DEFAULT_CONFIG.copy()
+        # track mtime
         try:
             self._last_mtime = os.path.getmtime(self.path)
         except Exception:
@@ -68,42 +74,81 @@ class ConfigManager:
         return False
 
 CONFIG = ConfigManager(CONFIG_PATH)
+# Purge saved regions on startup (fresh run)
+CONFIG.data['regions'] = {}
+CONFIG._write()
 # //////////////////////////////////////////////////////////////////////////
 # END CORE SECTION: CONFIG MANAGER
 # //////////////////////////////////////////////////////////////////////////
 
+# Assign or persist agent_id
+host = socket.gethostname().lower()
+stored_id = CONFIG.data.get('agent_id')
+if stored_id:
+    AGENT_ID = stored_id
+else:
+    AGENT_ID = f"{host}-agent-{uuid.uuid4().hex[:8]}"
+    CONFIG.data['agent_id'] = AGENT_ID
+    CONFIG._write()
+
 # //////////////////////////////////////////////////////////////////////////
 # CORE SECTION: WEBSOCKET SETUP & HANDLERS (do not modify unless absolutely necessary)
 # //////////////////////////////////////////////////////////////////////////
-AGENT_ID = f"{socket.gethostname().lower()}-agent-{uuid.uuid4().hex[:8]}"
 
 sio = socketio.AsyncClient(reconnection=True, reconnection_attempts=0, reconnection_delay=5)
 role_tasks = {}
+overlay_widgets = {}
 
 @sio.event
 async def connect():
-    print(f"[WebSocket] Agent ID: {AGENT_ID} connected to Borealis.")
+    print(f"[WebSocket] Connected to Agent ID: {AGENT_ID}.")
     await sio.emit('connect_agent', {"agent_id": AGENT_ID})
     await sio.emit('request_config', {"agent_id": AGENT_ID})
 
 @sio.event
 async def disconnect():
-    print("[WebSocket] Lost connection to Borealis server.")
+    print("[WebSocket] Disconnected from Borealis server.")
-
+    # reset tasks and overlays
-@sio.on('agent_config')
+    for task in list(role_tasks.values()):
-async def on_agent_config(cfg):
+        task.cancel()
-    print("[PROVISIONED] Received new configuration from Borealis.")
+    role_tasks.clear()
-    # cancel existing role tasks
+    for widget in list(overlay_widgets.values()):
+        try: widget.close()
+        except: pass
+    overlay_widgets.clear()
+    # purge regions on intentional disconnect
+    CONFIG.data['regions'].clear()
+    CONFIG._write()
+    # reload settings
+    CONFIG.load()
+
+@sio.on('agent_config')
+async def on_agent_config(cfg):
+    print(f"[CONNECTED] Received config with {len(cfg.get('roles',[]))} roles.")
+    # determine removed roles
+    new_ids = {r.get('node_id') for r in cfg.get('roles', []) if r.get('node_id')}
+    old_ids = set(role_tasks.keys())
+    removed = old_ids - new_ids
+    for rid in removed:
+        # remove region config
+        if rid in CONFIG.data['regions']:
+            CONFIG.data['regions'].pop(rid, None)
+        # close overlay
+        w = overlay_widgets.pop(rid, None)
+        if w:
+            try: w.close()
+            except: pass
+    if removed:
+        CONFIG._write()
+    # cancel existing and start new
     for task in list(role_tasks.values()):
         task.cancel()
     role_tasks.clear()
-    # start new tasks
     for role_cfg in cfg.get('roles', []):
-        role = role_cfg.get('role')
+        if role_cfg.get('role') == 'screenshot':
-        node_id = role_cfg.get('node_id')
+            nid = role_cfg.get('node_id')
-        if role == 'screenshot' and node_id:
+            t = asyncio.create_task(screenshot_task(role_cfg))
-            task = asyncio.create_task(screenshot_task(role_cfg))
+            role_tasks[nid] = t
-            role_tasks[node_id] = task
 # //////////////////////////////////////////////////////////////////////////
 # END CORE SECTION: WEBSOCKET SETUP & HANDLERS
 # //////////////////////////////////////////////////////////////////////////
@@ -120,7 +165,6 @@ class ScreenshotRegion(QtWidgets.QWidget):
         self.resizing = False
         self.resize_handle_size = 12
         self.setVisible(True)
-
         self.label = QtWidgets.QLabel(self)
         self.label.setText(f"{node_id[:8]}")
         self.label.setStyleSheet("color: lime; background: transparent; font-size: 10px;")
@@ -128,93 +172,73 @@ class ScreenshotRegion(QtWidgets.QWidget):
         self.setMouseTracking(True)
 
     def paintEvent(self, event):
-        painter = QtGui.QPainter(self)
+        p = QtGui.QPainter(self)
-        painter.setRenderHint(QtGui.QPainter.Antialiasing)
+        p.setRenderHint(QtGui.QPainter.Antialiasing)
-        painter.setBrush(QtCore.Qt.transparent)
+        p.setBrush(QtCore.Qt.transparent)
-        painter.setPen(QtGui.QPen(QtGui.QColor(0, 255, 0), 2))
+        p.setPen(QtGui.QPen(QtGui.QColor(0,255,0),2))
-        painter.drawRect(self.rect())
+        p.drawRect(self.rect())
-        handle = QtCore.QRect(self.width() - self.resize_handle_size,
+        hr = self.resize_handle_size
-                              self.height() - self.resize_handle_size,
+        hrect = QtCore.QRect(self.width()-hr, self.height()-hr, hr, hr)
-                              self.resize_handle_size, self.resize_handle_size)
+        p.fillRect(hrect, QtGui.QColor(0,255,0))
-        painter.fillRect(handle, QtGui.QColor(0, 255, 0))
 
-    def mousePressEvent(self, event):
+    def mousePressEvent(self, e):
-        if event.button() == QtCore.Qt.LeftButton:
+        if e.button()==QtCore.Qt.LeftButton:
-            px, py = event.pos().x(), event.pos().y()
+            x,y = e.pos().x(), e.pos().y()
-            if px > self.width() - self.resize_handle_size and \
+            if x>self.width()-self.resize_handle_size and y>self.height()-self.resize_handle_size:
-               py > self.height() - self.resize_handle_size:
                 self.resizing=True
             else:
-                self.drag_offset = event.globalPos() - self.frameGeometry().topLeft()
+                self.drag_offset = e.globalPos() - self.frameGeometry().topLeft()
 
-    def mouseMoveEvent(self, event):
+    def mouseMoveEvent(self, e):
         if self.resizing:
-            nw = max(event.pos().x(), 100)
+            nw = max(e.pos().x(),100)
-            nh = max(event.pos().y(), 80)
+            nh = max(e.pos().y(),80)
             self.resize(nw,nh)
-        elif event.buttons() & QtCore.Qt.LeftButton and self.drag_offset:
+        elif e.buttons()&QtCore.Qt.LeftButton and self.drag_offset:
-            self.move(event.globalPos() - self.drag_offset)
+            self.move(e.globalPos()-self.drag_offset)
 
-    def mouseReleaseEvent(self, event):
+    def mouseReleaseEvent(self,e):
-        self.resizing = False
+        self.resizing=False; self.drag_offset=None
-        self.drag_offset = None
 
     def get_geometry(self):
-        geo = self.geometry()
+        g=self.geometry(); return (g.x(),g.y(),g.width(),g.height())
-        return geo.x(), geo.y(), geo.width(), geo.height()
-
-# ---------------- Helper Functions ----------------
-app = None
-overlay_widgets = {}
-
-def create_overlay(node_id, region):
-    if node_id in overlay_widgets:
-        return
-    x, y, w, h = region
-    widget = ScreenshotRegion(node_id, x, y, w, h)
-    overlay_widgets[node_id] = widget
-    widget.show()
-
-def get_overlay_geometry(node_id):
-    widget = overlay_widgets.get(node_id)
-    if widget:
-        return widget.get_geometry()
-    return (0, 0, 0, 0)
 
 # ---------------- Screenshot Task ----------------
 async def screenshot_task(cfg):
+    nid = cfg.get('node_id');
+    # initial region from config or payload
+    r = CONFIG.data['regions'].get(nid)
+    region = (r['x'],r['y'],r['w'],r['h']) if r else (cfg.get('x',100),cfg.get('y',100),cfg.get('w',300),cfg.get('h',200))
+    if nid not in overlay_widgets:
+        widget = ScreenshotRegion(nid,*region)
+        overlay_widgets[nid] = widget; widget.show()
     interval = cfg.get('interval',1000)/1000.0
-    node_id = cfg.get('node_id')
-    region = (cfg.get('x', 100), cfg.get('y', 100), cfg.get('w', 300), cfg.get('h', 200))
-    create_overlay(node_id, region)
     loop = asyncio.get_event_loop()
-    executor = concurrent.futures.ThreadPoolExecutor(max_workers=CONFIG.data.get('max_workers', 8))
+    executor = concurrent.futures.ThreadPoolExecutor(max_workers=CONFIG.data.get('max_task_workers',DEFAULT_CONFIG['max_task_workers']))
     try:
         while True:
-            x, y, w, h = get_overlay_geometry(node_id)
+            x,y,w,h = overlay_widgets[nid].get_geometry()
+            # persist if changed
+            prev = CONFIG.data['regions'].get(nid)
+            if prev!={'x':x,'y':y,'w':w,'h':h}:
+                CONFIG.data['regions'][nid]={'x':x,'y':y,'w':w,'h':h}
+                CONFIG._write()
             grab = partial(ImageGrab.grab,bbox=(x,y,x+w,y+h))
             img = await loop.run_in_executor(executor,grab)
-            buf = BytesIO()
+            buf = BytesIO(); img.save(buf,format='PNG')
-            img.save(buf, format='PNG')
             encoded = base64.b64encode(buf.getvalue()).decode('utf-8')
-            await sio.emit('agent_screenshot_task', {
+            await sio.emit('agent_screenshot_task',{'agent_id':AGENT_ID,'node_id':nid,'image_base64':encoded})
-                'agent_id': AGENT_ID,
-                'node_id': node_id,
-                'image_base64': encoded
-            })
             await asyncio.sleep(interval)
     except asyncio.CancelledError:
         return
     except Exception as e:
-        print(f"[ERROR] Screenshot task {node_id} failed: {e}")
+        print(f"[ERROR] Screenshot task {nid} failed: {e}")
 
 # ---------------- Config Watcher ----------------
 async def config_watcher():
     while True:
-        if CONFIG.watch():
+        if CONFIG.watch(): pass
-            # settings updated, e.g., executor pool size will apply on next task run
+        await asyncio.sleep(CONFIG.data.get('config_file_watcher_interval',DEFAULT_CONFIG['config_file_watcher_interval']))
-            pass
-        await asyncio.sleep(CONFIG.data.get('config_watch_interval', 2))
 
 # //////////////////////////////////////////////////////////////////////////
 # CORE SECTION: MAIN & EVENT LOOP (do not modify unless you know what you’re doing)
@@ -223,11 +247,12 @@ async def connect_loop():
     retry=5
     while True:
         try:
-            print(f"[WebSocket] Connecting to {CONFIG.data['SERVER_URL']}...")
+            url=CONFIG.data.get('borealis_server_url',DEFAULT_CONFIG['borealis_server_url'])
-            await sio.connect(CONFIG.data['SERVER_URL'], transports=['websocket'])
+            print(f"[WebSocket] Connecting to {url}...")
+            await sio.connect(url,transports=['websocket'])
             break
-        except Exception:
+        except:
-            print(f"[WebSocket] Server not available, retrying in {retry}s...")
+            print(f"[WebSocket] Server unavailable, retrying in {retry}s...")
             await asyncio.sleep(retry)
 
 if __name__=='__main__':

Data/Server/WebUI/src/nodes/Agents/Node_Agent.jsx

@@ -1,81 +1,107 @@
 ////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/WebUI/src/nodes/Agent/Node_Agent.jsx
 
-import React, { useEffect, useState } from "react";
+import React, { useEffect, useState, useCallback, useMemo } from "react";
 import { Handle, Position, useReactFlow, useStore } from "reactflow";
 
 const BorealisAgentNode = ({ id, data }) => {
-    const { getNodes, getEdges, setNodes } = useReactFlow();
+  const { getNodes, setNodes } = useReactFlow();
-    const [agents, setAgents] = useState([]);
+  const edges = useStore((state) => state.edges);
+  const [agents, setAgents] = useState({});
   const [selectedAgent, setSelectedAgent] = useState(data.agent_id || "");
+  const [isConnected, setIsConnected] = useState(false);
 
-    // -------------------------------
+  // Build a normalized list [{id, status}, ...]
-    // Load agent list from backend
+  const agentList = useMemo(() => {
-    // -------------------------------
+    if (Array.isArray(agents)) {
+      return agents.map((a) => ({ id: a.id, status: a.status }));
+    } else if (agents && typeof agents === "object") {
+      return Object.entries(agents).map(([aid, info]) => ({ id: aid, status: info.status }));
+    }
+    return [];
+  }, [agents]);
+
+  // Fetch agents from backend
   useEffect(() => {
+    const fetchAgents = () => {
       fetch("/api/agents")
-            .then(res => res.json())
+        .then((res) => res.json())
-            .then(setAgents);
+        .then(setAgents)
-
+        .catch(() => {});
-        const interval = setInterval(() => {
+    };
-            fetch("/api/agents")
+    fetchAgents();
-                .then(res => res.json())
+    const interval = setInterval(fetchAgents, 5000);
-                .then(setAgents);
-        }, 5000);
-
     return () => clearInterval(interval);
   }, []);
 
-    // -------------------------------
+  // Persist selectedAgent and reset connection on change
-    // Helper: Get all provisioner role nodes connected to bottom port
+  useEffect(() => {
-    // -------------------------------
+    setNodes((nds) =>
-    const getAttachedProvisioners = () => {
+      nds.map((n) =>
+        n.id === id ? { ...n, data: { ...n.data, agent_id: selectedAgent } } : n
+      )
+    );
+    setIsConnected(false);
+  }, [selectedAgent]);
+
+  // Compute attached role node IDs for this agent node
+  const attachedRoleIds = useMemo(
+    () =>
+      edges
+        .filter((e) => e.source === id && e.sourceHandle === "provisioner")
+        .map((e) => e.target),
+    [edges, id]
+  );
+
+  // Build role payloads using the instruction registry
+  const getAttachedRoles = useCallback(() => {
     const allNodes = getNodes();
-        const allEdges = getEdges();
+    return attachedRoleIds
-        const attached = [];
+      .map((nid) => {
+        const fn = window.__BorealisInstructionNodes?.[nid];
+        return typeof fn === "function" ? fn() : null;
+      })
+      .filter((r) => r);
+  }, [attachedRoleIds, getNodes]);
 
-        for (const edge of allEdges) {
+  // Connect: send roles to server
-            if (edge.source === id && edge.sourceHandle === "provisioner") {
+  const handleConnect = useCallback(() => {
-                const roleNode = allNodes.find(n => n.id === edge.target);
-                if (roleNode && typeof window.__BorealisInstructionNodes?.[roleNode.id] === "function") {
-                    attached.push(window.__BorealisInstructionNodes[roleNode.id]());
-                }
-            }
-        }
-
-        return attached;
-    };
-
-    // -------------------------------
-    // Provision Agent with all Roles
-    // -------------------------------
-    const handleProvision = () => {
     if (!selectedAgent) return;
-
+    const roles = getAttachedRoles();
-        const provisionRoles = getAttachedProvisioners();
-        if (!provisionRoles.length) {
-            console.warn("No provisioner nodes connected to agent.");
-            return;
-        }
-
-        const configPayload = {
-            agent_id: selectedAgent,
-            roles: provisionRoles
-        };
-
     fetch("/api/agent/provision", {
       method: "POST",
       headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(configPayload)
+      body: JSON.stringify({ agent_id: selectedAgent, roles }),
     })
-            .then(res => res.json())
+      .then(() => setIsConnected(true))
-            .then(() => {
+      .catch(() => {});
-                console.log(`[Provision] Agent ${selectedAgent} updated with ${provisionRoles.length} roles.`);
+  }, [selectedAgent, getAttachedRoles]);
-            });
+
-    };
+  // Disconnect: clear roles on server
+  const handleDisconnect = useCallback(() => {
+    if (!selectedAgent) return;
+    fetch("/api/agent/provision", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ agent_id: selectedAgent, roles: [] }),
+    })
+      .then(() => setIsConnected(false))
+      .catch(() => {});
+  }, [selectedAgent]);
+
+  // Hot-update roles when attachedRoleIds change
+  useEffect(() => {
+    if (isConnected) {
+      const roles = getAttachedRoles();
+      fetch("/api/agent/provision", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ agent_id: selectedAgent, roles }),
+      }).catch(() => {});
+    }
+  }, [attachedRoleIds]);
 
   return (
     <div className="borealis-node">
-            {/* This bottom port is used for bi-directional provisioning & feedback */}
       <Handle
         type="source"
         position={Position.Bottom}
@@ -89,42 +115,41 @@ const BorealisAgentNode = ({ id, data }) => {
         <label>Agent:</label>
         <select
           value={selectedAgent}
-                    onChange={(e) => {
+          onChange={(e) => setSelectedAgent(e.target.value)}
-                        const newId = e.target.value;
-                        setSelectedAgent(newId);
-                        setNodes(nds =>
-                            nds.map(n =>
-                                n.id === id
-                                    ? { ...n, data: { ...n.data, agent_id: newId } }
-                                    : n
-                            )
-                        );
-                    }}
           style={{ width: "100%", marginBottom: "6px", fontSize: "9px" }}
         >
           <option value="">-- Select --</option>
-                    {Object.entries(agents).map(([id, info]) => {
+          {agentList.map(({ id: aid, status }) => {
-                        const label = info.status === "provisioned" ? "(Provisioned)" : "(Not Provisioned)";
+            const labelText = status === "provisioned" ? "(Connected)" : "(Ready to Connect)";
             return (
-                            <option key={id} value={id}>
+              <option key={aid} value={aid}>
-                                {id} {label}
+                {aid} {labelText}
               </option>
             );
           })}
         </select>
 
+        {isConnected ? (
           <button
-                    onClick={handleProvision}
+            onClick={handleDisconnect}
             style={{ width: "100%", fontSize: "9px", padding: "4px", marginTop: "4px" }}
           >
-                    Provision Agent
+            Disconnect from Agent
           </button>
+        ) : (
+          <button
+            onClick={handleConnect}
+            style={{ width: "100%", fontSize: "9px", padding: "4px", marginTop: "4px" }}
+            disabled={!selectedAgent}
+          >
+            Connect to Agent
+          </button>
+        )}
 
         <hr style={{ margin: "6px 0", borderColor: "#444" }} />
 
         <div style={{ fontSize: "8px", color: "#aaa" }}>
-                    Connect <strong>Agent Role Nodes</strong> below to define roles for this agent.
+          Attach <strong>Agent Role Nodes</strong> to define roles for this agent. Roles will be provisioned automatically.
-                    Each connected <strong>Agent Role Node</strong> will send back its collected data (like screenshots) through the role node itself and act as a separate data output.
         </div>
       </div>
     </div>
@@ -138,9 +163,9 @@ export default {
 Main Agent Node
 
 - Selects an available agent
-- Connect role nodes below to assign tasks to the agent
+- Connect/disconnects via button
-- Roles include screenshots, keyboard macros, etc.
+- Auto-updates roles when attached roles change
 `.trim(),
-    content: "Select and provision a Borealis Agent with task roles",
+  content: "Select and manage an Agent with dynamic roles",
-    component: BorealisAgentNode
+  component: BorealisAgentNode,
 };