Agent Reverse Tunneling - Engine Tunnel Service Implementation

Data/Engine/services/API/__init__.py

@@ -32,6 +32,7 @@ from ...integrations import GitHubIntegration
 from ..auth import DevModeManager
 from .enrollment import routes as enrollment_routes
 from .tokens import routes as token_routes
+from .devices.tunnel import register_tunnel
 
 from ...server import EngineContext
 from .access_management.login import register_auth
@@ -285,6 +286,7 @@ def _register_devices(app: Flask, adapters: EngineServiceAdapters) -> None:
     register_management(app, adapters)
     register_admin_endpoints(app, adapters)
     device_routes.register_agents(app, adapters)
+    register_tunnel(app, adapters)
 
 def _register_filters(app: Flask, adapters: EngineServiceAdapters) -> None:
     filters_management.register_filters(app, adapters)

Data/Engine/services/API/devices/tunnel.py

@@ -0,0 +1,138 @@
+# ======================================================
+# Data\Engine\services\API\devices\tunnel.py
+# Description: Negotiation endpoint for reverse tunnel leases (operator-initiated; dormant until tunnel listener is wired).
+#
+# API Endpoints (if applicable):
+# - POST /api/tunnel/request (Token Authenticated) - Allocates a reverse tunnel lease for the requested agent/protocol.
+# ======================================================
+
+"""Reverse tunnel negotiation API (Engine side)."""
+from __future__ import annotations
+
+import os
+from typing import Any, Dict, Optional, Tuple
+
+from flask import Blueprint, jsonify, request, session
+from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer
+
+from ...WebSocket.Agent.ReverseTunnel import ReverseTunnelService
+
+if False:  # pragma: no cover - import cycle hint for type checkers
+    from .. import EngineServiceAdapters
+
+
+def _current_user(app) -> Optional[Dict[str, str]]:
+    """Resolve operator identity from session or signed token."""
+
+    username = session.get("username")
+    role = session.get("role") or "User"
+    if username:
+        return {"username": username, "role": role}
+
+    token = None
+    auth_header = request.headers.get("Authorization") or ""
+    if auth_header.lower().startswith("bearer "):
+        token = auth_header.split(" ", 1)[1].strip()
+    if not token:
+        token = request.cookies.get("borealis_auth")
+    if not token:
+        return None
+
+    try:
+        serializer = URLSafeTimedSerializer(app.secret_key or "borealis-dev-secret", salt="borealis-auth")
+        token_ttl = int(os.environ.get("BOREALIS_TOKEN_TTL_SECONDS", 60 * 60 * 24 * 30))
+        data = serializer.loads(token, max_age=token_ttl)
+        username = data.get("u")
+        role = data.get("r") or "User"
+        if username:
+            return {"username": username, "role": role}
+    except (BadSignature, SignatureExpired, Exception):
+        return None
+    return None
+
+
+def _require_login(app) -> Optional[Tuple[Dict[str, Any], int]]:
+    user = _current_user(app)
+    if not user:
+        return {"error": "unauthorized"}, 401
+    return None
+
+
+def _get_tunnel_service(adapters: "EngineServiceAdapters") -> ReverseTunnelService:
+    service = getattr(adapters.context, "reverse_tunnel_service", None) or getattr(adapters, "_reverse_tunnel_service", None)
+    if service is None:
+        service = ReverseTunnelService(
+            adapters.context,
+            signer=getattr(adapters, "script_signer", None),
+            db_conn_factory=adapters.db_conn_factory,
+            socketio=getattr(adapters.context, "socketio", None),
+        )
+        service.start()
+        setattr(adapters, "_reverse_tunnel_service", service)
+        setattr(adapters.context, "reverse_tunnel_service", service)
+    return service
+
+
+def _normalize_text(value: Any) -> str:
+    if value is None:
+        return ""
+    try:
+        return str(value).strip()
+    except Exception:
+        return ""
+
+
+def register_tunnel(app, adapters: "EngineServiceAdapters") -> None:
+    """Register reverse tunnel negotiation endpoints."""
+
+    blueprint = Blueprint("reverse_tunnel", __name__)
+    service_log = adapters.service_log
+    logger = adapters.context.logger.getChild("tunnel.api")
+
+    @blueprint.route("/api/tunnel/request", methods=["POST"])
+    def request_tunnel():
+        requirement = _require_login(app)
+        if requirement:
+            payload, status = requirement
+            return jsonify(payload), status
+
+        user = _current_user(app) or {}
+        operator_id = user.get("username") or None
+
+        body = request.get_json(silent=True) or {}
+        agent_id = _normalize_text(body.get("agent_id"))
+        protocol = _normalize_text(body.get("protocol") or "ps").lower() or "ps"
+        domain = _normalize_text(body.get("domain") or protocol).lower() or protocol
+
+        if not agent_id:
+            return jsonify({"error": "agent_id_required"}), 400
+
+        tunnel_service = _get_tunnel_service(adapters)
+        try:
+            lease = tunnel_service.request_lease(
+                agent_id=agent_id,
+                protocol=protocol,
+                domain=domain,
+                operator_id=operator_id,
+            )
+        except RuntimeError as exc:
+            message = str(exc)
+            if message.startswith("domain_limit:"):
+                domain_name = message.split(":", 1)[-1] if ":" in message else domain
+                return jsonify({"error": "domain_limit", "domain": domain_name}), 409
+            if message == "port_pool_exhausted":
+                return jsonify({"error": "port_pool_exhausted"}), 503
+            logger.warning("tunnel lease request failed for agent_id=%s: %s", agent_id, message)
+            return jsonify({"error": "lease_allocation_failed"}), 500
+
+        summary = tunnel_service.lease_summary(lease)
+        summary["fixed_port"] = tunnel_service.fixed_port
+        summary["heartbeat_seconds"] = tunnel_service.heartbeat_seconds
+
+        service_log(
+            "reverse_tunnel",
+            f"lease created tunnel_id={lease.tunnel_id} agent_id={lease.agent_id} domain={lease.domain} protocol={lease.protocol}",
+        )
+        return jsonify(summary), 200
+
+    app.register_blueprint(blueprint)

Data/Engine/services/WebSocket/Agent/__init__.py

@@ -0,0 +1,10 @@
+# ======================================================
+# Data\Engine\services\WebSocket\Agent\__init__.py
+# Description: Package marker for Agent-facing WebSocket services (reverse tunnel scaffolding).
+#
+# API Endpoints (if applicable): None
+# ======================================================
+
+"""Agent-facing WebSocket services for the Engine runtime."""
+
+__all__ = []

Data/Engine/services/WebSocket/__init__.py

@@ -8,6 +8,7 @@
 """WebSocket service registration for the Borealis Engine runtime."""
 from __future__ import annotations
 
+import base64
 import sqlite3
 import time
 from dataclasses import dataclass, field
@@ -15,9 +16,16 @@ from pathlib import Path
 from typing import Any, Callable, Dict, Optional
 
 from flask_socketio import SocketIO
+from flask import session, request
 
 from ...database import initialise_engine_database
 from ...server import EngineContext
+from .Agent.ReverseTunnel import (
+    ReverseTunnelService,
+    TunnelBridge,
+    decode_frame,
+    TunnelFrame,
+)
 from ..API import _make_db_conn_factory, _make_service_logger
 
 
@@ -63,6 +71,16 @@ def register_realtime(socket_server: SocketIO, context: EngineContext) -> None:
 
     adapters = EngineRealtimeAdapters(context)
     logger = context.logger.getChild("realtime.quick_jobs")
+    tunnel_service = getattr(context, "reverse_tunnel_service", None)
+    if tunnel_service is None:
+        tunnel_service = ReverseTunnelService(
+            context,
+            signer=None,
+            db_conn_factory=adapters.db_conn_factory,
+            socketio=socket_server,
+        )
+        tunnel_service.start()
+        setattr(context, "reverse_tunnel_service", tunnel_service)
 
     @socket_server.on("quick_job_result")
     def _handle_quick_job_result(data: Any) -> None:
@@ -224,3 +242,163 @@ def register_realtime(socket_server: SocketIO, context: EngineContext) -> None:
                     job_id,
                     exc,
                 )
+
+    @socket_server.on("tunnel_bridge_attach")
+    def _tunnel_bridge_attach(data: Any) -> Any:
+        """Placeholder operator bridge attach handler (no data channel yet)."""
+
+        if not isinstance(data, dict):
+            return {"error": "invalid_payload"}
+
+        tunnel_id = str(data.get("tunnel_id") or "").strip()
+        operator_id = str(data.get("operator_id") or "").strip() or None
+        if not tunnel_id:
+            return {"error": "tunnel_id_required"}
+
+        try:
+            tunnel_service.operator_attach(tunnel_id, operator_id)
+        except ValueError as exc:
+            return {"error": str(exc)}
+        except Exception as exc:  # pragma: no cover - defensive guard
+            logger.debug("tunnel_bridge_attach failed tunnel_id=%s: %s", tunnel_id, exc, exc_info=True)
+            return {"error": "bridge_attach_failed"}
+
+        return {"status": "ok", "tunnel_id": tunnel_id, "operator_id": operator_id or "-"}
+
+    def _encode_frame(frame: TunnelFrame) -> str:
+        return base64.b64encode(frame.encode()).decode("ascii")
+
+    def _decode_frame_payload(raw: Any) -> TunnelFrame:
+        if isinstance(raw, str):
+            try:
+                raw_bytes = base64.b64decode(raw)
+            except Exception:
+                raise ValueError("invalid_frame")
+        elif isinstance(raw, (bytes, bytearray)):
+            raw_bytes = bytes(raw)
+        else:
+            raise ValueError("invalid_frame")
+        return decode_frame(raw_bytes)
+
+    @socket_server.on("tunnel_operator_send")
+    def _tunnel_operator_send(data: Any) -> Any:
+        """Operator -> agent frame enqueue (placeholder queue)."""
+
+        if not isinstance(data, dict):
+            return {"error": "invalid_payload"}
+        tunnel_id = str(data.get("tunnel_id") or "").strip()
+        frame_raw = data.get("frame")
+        if not tunnel_id or frame_raw is None:
+            return {"error": "tunnel_id_and_frame_required"}
+        try:
+            frame = _decode_frame_payload(frame_raw)
+        except Exception as exc:
+            return {"error": str(exc)}
+
+        bridge: Optional[TunnelBridge] = tunnel_service.get_bridge(tunnel_id)
+        if bridge is None:
+            return {"error": "unknown_tunnel"}
+        bridge.operator_to_agent(frame)
+        return {"status": "ok"}
+
+    @socket_server.on("tunnel_operator_poll")
+    def _tunnel_operator_poll(data: Any) -> Any:
+        """Operator polls queued frames from agent."""
+
+        tunnel_id = ""
+        if isinstance(data, dict):
+            tunnel_id = str(data.get("tunnel_id") or "").strip()
+        if not tunnel_id:
+            return {"error": "tunnel_id_required"}
+        bridge: Optional[TunnelBridge] = tunnel_service.get_bridge(tunnel_id)
+        if bridge is None:
+            return {"error": "unknown_tunnel"}
+
+        frames = []
+        while True:
+            frame = bridge.next_for_operator()
+            if frame is None:
+                break
+            frames.append(_encode_frame(frame))
+        return {"frames": frames}
+
+    # WebUI operator bridge namespace for browser clients
+    tunnel_namespace = "/tunnel"
+    _operator_sessions: Dict[str, str] = {}
+
+    def _current_operator() -> Optional[str]:
+        username = session.get("username")
+        if username:
+            return str(username)
+        auth_header = (request.headers.get("Authorization") or "").strip()
+        token = None
+        if auth_header.lower().startswith("bearer "):
+            token = auth_header.split(" ", 1)[1].strip()
+        if not token:
+            token = request.cookies.get("borealis_auth")
+        return token or None
+
+    @socket_server.on("join", namespace=tunnel_namespace)
+    def _ws_tunnel_join(data: Any) -> Any:
+        if not isinstance(data, dict):
+            return {"error": "invalid_payload"}
+        operator_id = _current_operator()
+        if not operator_id:
+            return {"error": "unauthorized"}
+        tunnel_id = str(data.get("tunnel_id") or "").strip()
+        if not tunnel_id:
+            return {"error": "tunnel_id_required"}
+        bridge = tunnel_service.get_bridge(tunnel_id)
+        if bridge is None:
+            return {"error": "unknown_tunnel"}
+        try:
+            tunnel_service.operator_attach(tunnel_id, operator_id)
+        except Exception as exc:
+            logger.debug("ws_tunnel_join failed tunnel_id=%s: %s", tunnel_id, exc, exc_info=True)
+            return {"error": "attach_failed"}
+        sid = request.sid
+        _operator_sessions[sid] = tunnel_id
+        return {"status": "ok", "tunnel_id": tunnel_id}
+
+    @socket_server.on("send", namespace=tunnel_namespace)
+    def _ws_tunnel_send(data: Any) -> Any:
+        sid = request.sid
+        tunnel_id = _operator_sessions.get(sid)
+        if not tunnel_id:
+            return {"error": "not_joined"}
+        if not isinstance(data, dict):
+            return {"error": "invalid_payload"}
+        frame_raw = data.get("frame")
+        if frame_raw is None:
+            return {"error": "frame_required"}
+        try:
+            frame = _decode_frame_payload(frame_raw)
+        except Exception:
+            return {"error": "invalid_frame"}
+        bridge = tunnel_service.get_bridge(tunnel_id)
+        if bridge is None:
+            return {"error": "unknown_tunnel"}
+        bridge.operator_to_agent(frame)
+        return {"status": "ok"}
+
+    @socket_server.on("poll", namespace=tunnel_namespace)
+    def _ws_tunnel_poll() -> Any:
+        sid = request.sid
+        tunnel_id = _operator_sessions.get(sid)
+        if not tunnel_id:
+            return {"error": "not_joined"}
+        bridge = tunnel_service.get_bridge(tunnel_id)
+        if bridge is None:
+            return {"error": "unknown_tunnel"}
+        frames = []
+        while True:
+            frame = bridge.next_for_operator()
+            if frame is None:
+                break
+            frames.append(_encode_frame(frame))
+        return {"frames": frames}
+
+    @socket_server.on("disconnect", namespace=tunnel_namespace)
+    def _ws_tunnel_disconnect():
+        sid = request.sid
+        _operator_sessions.pop(sid, None)