Overhaul of Reverse Tunnel Code

This commit is contained in:
2025-12-06 20:07:08 -07:00
parent 737bf1faef
commit 178257c588
42 changed files with 1240 additions and 357 deletions

View File

@@ -0,0 +1,3 @@
"""Namespace package for reverse tunnel domains (Agent side)."""
__all__ = ["remote_interactive_shell", "remote_management", "remote_video"]

View File

@@ -0,0 +1,49 @@
"""Placeholder Bash channel (Agent side)."""
from __future__ import annotations
import asyncio
from typing import Any, Dict, Optional
MSG_DATA = 0x05
MSG_CONTROL = 0x09
MSG_CLOSE = 0x08
CLOSE_PROTOCOL_ERROR = 3
CLOSE_AGENT_SHUTDOWN = 6
class BashChannel:
"""Stub Bash handler that immediately reports unsupported."""
def __init__(self, role, tunnel, channel_id: int, metadata: Optional[Dict[str, Any]]):
self.role = role
self.tunnel = tunnel
self.channel_id = channel_id
self.metadata = metadata or {}
self.loop = getattr(role, "loop", None) or asyncio.get_event_loop()
self._closed = False
async def start(self) -> None:
# Until Bash support is implemented, close the channel to free resources.
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="bash_unsupported")
async def on_frame(self, frame) -> None:
if self._closed:
return
if frame.msg_type in (MSG_DATA, MSG_CONTROL):
# Ignore payloads but acknowledge by stopping the channel to avoid leaks.
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="bash_unsupported")
elif frame.msg_type == MSG_CLOSE:
await self.stop(code=CLOSE_AGENT_SHUTDOWN, reason="operator_close")
async def stop(self, code: int = CLOSE_PROTOCOL_ERROR, reason: str = "") -> None:
if self._closed:
return
self._closed = True
try:
await self.role._send_frame(self.tunnel, self.role.close_frame(self.channel_id, code, reason or "bash_closed"))
except Exception:
pass
self.role._log(f"reverse_tunnel bash channel stopped channel={self.channel_id} reason={reason or 'closed'}")
__all__ = ["BashChannel"]

View File

@@ -0,0 +1,35 @@
"""Expose the PowerShell channel under the domain path, with file-based import fallback."""
from __future__ import annotations
import importlib.util
from pathlib import Path
powershell_module = None
# Attempt package-relative import first
try: # pragma: no cover - best effort
from ....ReverseTunnel import tunnel_Powershell as powershell_module # type: ignore
except Exception:
powershell_module = None
# Fallback: load directly from file path to survive non-package runtimes
if powershell_module is None:
try:
base = Path(__file__).resolve().parents[3] / "ReverseTunnel" / "tunnel_Powershell.py"
spec = importlib.util.spec_from_file_location("tunnel_Powershell", base)
if spec and spec.loader:
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) # type: ignore
powershell_module = module
except Exception:
powershell_module = None
if powershell_module and hasattr(powershell_module, "PowershellChannel"):
PowershellChannel = powershell_module.PowershellChannel # type: ignore
else: # pragma: no cover - safety guard
class PowershellChannel: # type: ignore
def __init__(self, *args, **kwargs):
raise ImportError("PowerShell channel unavailable")
__all__ = ["PowershellChannel"]

View File

@@ -0,0 +1,6 @@
"""Protocol handlers for interactive shell tunnels (Agent side)."""
from .Powershell import PowershellChannel
from .Bash import BashChannel
__all__ = ["PowershellChannel", "BashChannel"]

View File

@@ -0,0 +1,3 @@
"""Interactive shell domain (PowerShell/Bash) handlers."""
__all__ = ["tunnel", "Protocols"]

View File

@@ -0,0 +1,5 @@
"""Placeholder module for remote interactive shell tunnel domain (Agent side)."""
DOMAIN_NAME = "remote-interactive-shell"
__all__ = ["DOMAIN_NAME"]

View File

@@ -0,0 +1,47 @@
"""Placeholder SSH channel (Agent side)."""
from __future__ import annotations
import asyncio
from typing import Any, Dict, Optional
MSG_DATA = 0x05
MSG_CONTROL = 0x09
MSG_CLOSE = 0x08
CLOSE_PROTOCOL_ERROR = 3
CLOSE_AGENT_SHUTDOWN = 6
class SSHChannel:
"""Stub SSH handler that marks the channel unsupported for now."""
def __init__(self, role, tunnel, channel_id: int, metadata: Optional[Dict[str, Any]]):
self.role = role
self.tunnel = tunnel
self.channel_id = channel_id
self.metadata = metadata or {}
self.loop = getattr(role, "loop", None) or asyncio.get_event_loop()
self._closed = False
async def start(self) -> None:
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="ssh_unsupported")
async def on_frame(self, frame) -> None:
if self._closed:
return
if frame.msg_type in (MSG_DATA, MSG_CONTROL):
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="ssh_unsupported")
elif frame.msg_type == MSG_CLOSE:
await self.stop(code=CLOSE_AGENT_SHUTDOWN, reason="operator_close")
async def stop(self, code: int = CLOSE_PROTOCOL_ERROR, reason: str = "") -> None:
if self._closed:
return
self._closed = True
try:
await self.role._send_frame(self.tunnel, self.role.close_frame(self.channel_id, code, reason or "ssh_closed"))
except Exception:
pass
self.role._log(f"reverse_tunnel ssh channel stopped channel={self.channel_id} reason={reason or 'closed'}")
__all__ = ["SSHChannel"]

View File

@@ -0,0 +1,47 @@
"""Placeholder WinRM channel (Agent side)."""
from __future__ import annotations
import asyncio
from typing import Any, Dict, Optional
MSG_DATA = 0x05
MSG_CONTROL = 0x09
MSG_CLOSE = 0x08
CLOSE_PROTOCOL_ERROR = 3
CLOSE_AGENT_SHUTDOWN = 6
class WinRMChannel:
"""Stub WinRM handler that marks the channel unsupported for now."""
def __init__(self, role, tunnel, channel_id: int, metadata: Optional[Dict[str, Any]]):
self.role = role
self.tunnel = tunnel
self.channel_id = channel_id
self.metadata = metadata or {}
self.loop = getattr(role, "loop", None) or asyncio.get_event_loop()
self._closed = False
async def start(self) -> None:
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="winrm_unsupported")
async def on_frame(self, frame) -> None:
if self._closed:
return
if frame.msg_type in (MSG_DATA, MSG_CONTROL):
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="winrm_unsupported")
elif frame.msg_type == MSG_CLOSE:
await self.stop(code=CLOSE_AGENT_SHUTDOWN, reason="operator_close")
async def stop(self, code: int = CLOSE_PROTOCOL_ERROR, reason: str = "") -> None:
if self._closed:
return
self._closed = True
try:
await self.role._send_frame(self.tunnel, self.role.close_frame(self.channel_id, code, reason or "winrm_closed"))
except Exception:
pass
self.role._log(f"reverse_tunnel winrm channel stopped channel={self.channel_id} reason={reason or 'closed'}")
__all__ = ["WinRMChannel"]

View File

@@ -0,0 +1,6 @@
"""Protocol handlers for remote management tunnels (Agent side)."""
from .SSH import SSHChannel
from .WinRM import WinRMChannel
__all__ = ["SSHChannel", "WinRMChannel"]

View File

@@ -0,0 +1,3 @@
"""Remote management domain (SSH/WinRM) handlers."""
__all__ = ["tunnel", "Protocols"]

View File

@@ -0,0 +1,5 @@
"""Placeholder module for remote management domain (Agent side)."""
DOMAIN_NAME = "remote-management"
__all__ = ["DOMAIN_NAME"]

View File

@@ -0,0 +1,47 @@
"""Placeholder RDP channel (Agent side)."""
from __future__ import annotations
import asyncio
from typing import Any, Dict, Optional
MSG_DATA = 0x05
MSG_CONTROL = 0x09
MSG_CLOSE = 0x08
CLOSE_PROTOCOL_ERROR = 3
CLOSE_AGENT_SHUTDOWN = 6
class RDPChannel:
"""Stub RDP handler that marks the channel unsupported for now."""
def __init__(self, role, tunnel, channel_id: int, metadata: Optional[Dict[str, Any]]):
self.role = role
self.tunnel = tunnel
self.channel_id = channel_id
self.metadata = metadata or {}
self.loop = getattr(role, "loop", None) or asyncio.get_event_loop()
self._closed = False
async def start(self) -> None:
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="rdp_unsupported")
async def on_frame(self, frame) -> None:
if self._closed:
return
if frame.msg_type in (MSG_DATA, MSG_CONTROL):
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="rdp_unsupported")
elif frame.msg_type == MSG_CLOSE:
await self.stop(code=CLOSE_AGENT_SHUTDOWN, reason="operator_close")
async def stop(self, code: int = CLOSE_PROTOCOL_ERROR, reason: str = "") -> None:
if self._closed:
return
self._closed = True
try:
await self.role._send_frame(self.tunnel, self.role.close_frame(self.channel_id, code, reason or "rdp_closed"))
except Exception:
pass
self.role._log(f"reverse_tunnel rdp channel stopped channel={self.channel_id} reason={reason or 'closed'}")
__all__ = ["RDPChannel"]

View File

@@ -0,0 +1,47 @@
"""Placeholder VNC channel (Agent side)."""
from __future__ import annotations
import asyncio
from typing import Any, Dict, Optional
MSG_DATA = 0x05
MSG_CONTROL = 0x09
MSG_CLOSE = 0x08
CLOSE_PROTOCOL_ERROR = 3
CLOSE_AGENT_SHUTDOWN = 6
class VNCChannel:
"""Stub VNC handler that marks the channel unsupported for now."""
def __init__(self, role, tunnel, channel_id: int, metadata: Optional[Dict[str, Any]]):
self.role = role
self.tunnel = tunnel
self.channel_id = channel_id
self.metadata = metadata or {}
self.loop = getattr(role, "loop", None) or asyncio.get_event_loop()
self._closed = False
async def start(self) -> None:
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="vnc_unsupported")
async def on_frame(self, frame) -> None:
if self._closed:
return
if frame.msg_type in (MSG_DATA, MSG_CONTROL):
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="vnc_unsupported")
elif frame.msg_type == MSG_CLOSE:
await self.stop(code=CLOSE_AGENT_SHUTDOWN, reason="operator_close")
async def stop(self, code: int = CLOSE_PROTOCOL_ERROR, reason: str = "") -> None:
if self._closed:
return
self._closed = True
try:
await self.role._send_frame(self.tunnel, self.role.close_frame(self.channel_id, code, reason or "vnc_closed"))
except Exception:
pass
self.role._log(f"reverse_tunnel vnc channel stopped channel={self.channel_id} reason={reason or 'closed'}")
__all__ = ["VNCChannel"]

View File

@@ -0,0 +1,47 @@
"""Placeholder WebRTC channel (Agent side)."""
from __future__ import annotations
import asyncio
from typing import Any, Dict, Optional
MSG_DATA = 0x05
MSG_CONTROL = 0x09
MSG_CLOSE = 0x08
CLOSE_PROTOCOL_ERROR = 3
CLOSE_AGENT_SHUTDOWN = 6
class WebRTCChannel:
"""Stub WebRTC handler that marks the channel unsupported for now."""
def __init__(self, role, tunnel, channel_id: int, metadata: Optional[Dict[str, Any]]):
self.role = role
self.tunnel = tunnel
self.channel_id = channel_id
self.metadata = metadata or {}
self.loop = getattr(role, "loop", None) or asyncio.get_event_loop()
self._closed = False
async def start(self) -> None:
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="webrtc_unsupported")
async def on_frame(self, frame) -> None:
if self._closed:
return
if frame.msg_type in (MSG_DATA, MSG_CONTROL):
await self.stop(code=CLOSE_PROTOCOL_ERROR, reason="webrtc_unsupported")
elif frame.msg_type == MSG_CLOSE:
await self.stop(code=CLOSE_AGENT_SHUTDOWN, reason="operator_close")
async def stop(self, code: int = CLOSE_PROTOCOL_ERROR, reason: str = "") -> None:
if self._closed:
return
self._closed = True
try:
await self.role._send_frame(self.tunnel, self.role.close_frame(self.channel_id, code, reason or "webrtc_closed"))
except Exception:
pass
self.role._log(f"reverse_tunnel webrtc channel stopped channel={self.channel_id} reason={reason or 'closed'}")
__all__ = ["WebRTCChannel"]

View File

@@ -0,0 +1,7 @@
"""Protocol handlers for remote video tunnels (Agent side)."""
from .WebRTC import WebRTCChannel
from .RDP import RDPChannel
from .VNC import VNCChannel
__all__ = ["WebRTCChannel", "RDPChannel", "VNCChannel"]

View File

@@ -0,0 +1,3 @@
"""Remote video/desktop domain (RDP/VNC/WebRTC) handlers."""
__all__ = ["tunnel", "Protocols"]

View File

@@ -0,0 +1,5 @@
"""Placeholder module for remote video domain (Agent side)."""
DOMAIN_NAME = "remote-video"
__all__ = ["DOMAIN_NAME"]

View File

@@ -14,25 +14,109 @@ import aiohttp
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import ed25519
# Capture import errors for the PowerShell handler so we can report why it is missing.
# Capture import errors for protocol handlers so we can report why they are missing.
PS_IMPORT_ERROR: Optional[str] = None
BASH_IMPORT_ERROR: Optional[str] = None
tunnel_SSH = None
tunnel_WinRM = None
tunnel_VNC = None
tunnel_RDP = None
tunnel_WebRTC = None
tunnel_Powershell = None
tunnel_Bash = None
def _load_protocol_module(module_name: str, rel_parts: list[str]) -> tuple[Optional[object], Optional[str]]:
"""Load a protocol handler directly from a file path to survive non-package runtimes."""
base = Path(__file__).parent
path = base
for part in rel_parts:
path = path / part
if not path.exists():
return None, f"path_missing:{path}"
try:
spec = importlib.util.spec_from_file_location(module_name, path)
if not spec or not spec.loader:
return None, "spec_failed"
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) # type: ignore
return module, None
except Exception as exc: # pragma: no cover - defensive
return None, repr(exc)
try:
from .ReverseTunnel import tunnel_Powershell # type: ignore
from .Reverse_Tunnels.remote_interactive_shell.Protocols import Powershell as tunnel_Powershell # type: ignore
except Exception as exc: # pragma: no cover - best-effort logging only
PS_IMPORT_ERROR = repr(exc)
# Try manual import from file to survive non-package execution.
try:
_ps_path = Path(__file__).parent / "ReverseTunnel" / "tunnel_Powershell.py"
if _ps_path.exists():
spec = importlib.util.spec_from_file_location("tunnel_Powershell", _ps_path)
if spec and spec.loader:
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) # type: ignore
tunnel_Powershell = module
PS_IMPORT_ERROR = None
except Exception as exc2: # pragma: no cover - diagnostic only
PS_IMPORT_ERROR = f"{PS_IMPORT_ERROR} | fallback_load_failed={exc2!r}"
_module, _err = _load_protocol_module(
"tunnel_Powershell",
["Reverse_Tunnels", "remote_interactive_shell", "Protocols", "Powershell.py"],
)
if _module:
tunnel_Powershell = _module
PS_IMPORT_ERROR = None
else:
try:
from .ReverseTunnel import tunnel_Powershell # type: ignore # legacy fallback
PS_IMPORT_ERROR = None
except Exception as exc2: # pragma: no cover - diagnostic only
PS_IMPORT_ERROR = f"{PS_IMPORT_ERROR} | legacy_fallback={exc2!r} | file_load_failed={_err}"
try:
from .Reverse_Tunnels.remote_interactive_shell.Protocols import Bash as tunnel_Bash # type: ignore
except Exception as exc: # pragma: no cover - best-effort logging only
BASH_IMPORT_ERROR = repr(exc)
_module, _err = _load_protocol_module(
"tunnel_Bash",
["Reverse_Tunnels", "remote_interactive_shell", "Protocols", "Bash.py"],
)
if _module:
tunnel_Bash = _module
BASH_IMPORT_ERROR = None
else:
BASH_IMPORT_ERROR = f"{BASH_IMPORT_ERROR} | file_load_failed={_err}"
try:
from .Reverse_Tunnels.remote_management.Protocols import SSH as tunnel_SSH # type: ignore
except Exception:
_module, _err = _load_protocol_module(
"tunnel_SSH",
["Reverse_Tunnels", "remote_management", "Protocols", "SSH.py"],
)
tunnel_SSH = _module
try:
from .Reverse_Tunnels.remote_management.Protocols import WinRM as tunnel_WinRM # type: ignore
except Exception:
_module, _err = _load_protocol_module(
"tunnel_WinRM",
["Reverse_Tunnels", "remote_management", "Protocols", "WinRM.py"],
)
tunnel_WinRM = _module
try:
from .Reverse_Tunnels.remote_video.Protocols import VNC as tunnel_VNC # type: ignore
except Exception:
_module, _err = _load_protocol_module(
"tunnel_VNC",
["Reverse_Tunnels", "remote_video", "Protocols", "VNC.py"],
)
tunnel_VNC = _module
try:
from .Reverse_Tunnels.remote_video.Protocols import RDP as tunnel_RDP # type: ignore
except Exception:
_module, _err = _load_protocol_module(
"tunnel_RDP",
["Reverse_Tunnels", "remote_video", "Protocols", "RDP.py"],
)
tunnel_RDP = _module
try:
from .Reverse_Tunnels.remote_video.Protocols import WebRTC as tunnel_WebRTC # type: ignore
except Exception:
_module, _err = _load_protocol_module(
"tunnel_WebRTC",
["Reverse_Tunnels", "remote_video", "Protocols", "WebRTC.py"],
)
tunnel_WebRTC = _module
ROLE_NAME = "reverse_tunnel"
ROLE_CONTEXTS = ["interactive", "system"]
@@ -185,10 +269,14 @@ class Role:
self._active: Dict[str, ActiveTunnel] = {}
self._domain_claims: Dict[str, str] = {}
self._domain_limits: Dict[str, Optional[int]] = {
"ps": 1,
"remote-interactive-shell": 2,
"remote-management": 1,
"remote-video": 2,
# Legacy / protocol fallbacks
"ps": 2,
"rdp": 1,
"vnc": 1,
"webrtc": 1,
"webrtc": 2,
"ssh": None,
"winrm": None,
}
@@ -211,6 +299,28 @@ class Role:
)
except Exception as exc:
self._log(f"reverse_tunnel ps handler registration failed: {exc}", error=True)
try:
if tunnel_Bash and hasattr(tunnel_Bash, "BashChannel"):
self._protocol_handlers["bash"] = tunnel_Bash.BashChannel
module_path = getattr(tunnel_Bash, "__file__", None)
self._log(f"reverse_tunnel bash handler registered (BashChannel) module={module_path}")
elif BASH_IMPORT_ERROR:
self._log(f"reverse_tunnel bash handler NOT registered (missing module/class) import_error={BASH_IMPORT_ERROR}", error=True)
except Exception as exc:
self._log(f"reverse_tunnel bash handler registration failed: {exc}", error=True)
try:
if tunnel_SSH and hasattr(tunnel_SSH, "SSHChannel"):
self._protocol_handlers["ssh"] = tunnel_SSH.SSHChannel
if tunnel_WinRM and hasattr(tunnel_WinRM, "WinRMChannel"):
self._protocol_handlers["winrm"] = tunnel_WinRM.WinRMChannel
if tunnel_VNC and hasattr(tunnel_VNC, "VNCChannel"):
self._protocol_handlers["vnc"] = tunnel_VNC.VNCChannel
if tunnel_RDP and hasattr(tunnel_RDP, "RDPChannel"):
self._protocol_handlers["rdp"] = tunnel_RDP.RDPChannel
if tunnel_WebRTC and hasattr(tunnel_WebRTC, "WebRTCChannel"):
self._protocol_handlers["webrtc"] = tunnel_WebRTC.WebRTCChannel
except Exception as exc:
self._log(f"reverse_tunnel protocol handler registration failed: {exc}", error=True)
# ------------------------------------------------------------------ Logging
def _log(self, message: str, *, error: bool = False) -> None:
@@ -728,6 +838,7 @@ class Role:
try:
handler = handler_cls(self, tunnel, frame.channel_id, metadata)
except Exception:
self._log(f"reverse_tunnel channel handler fallback to BaseChannel protocol={protocol}", error=True)
handler = BaseChannel(self, tunnel, frame.channel_id, metadata)
tunnel.channels[frame.channel_id] = handler
await handler.start()

View File

@@ -56,7 +56,7 @@ def test_reverse_tunnel_powershell_roundtrip() -> None:
# 1) Request a tunnel lease
resp = sess.post(
f"{HOST}/api/tunnel/request",
json={"agent_id": AGENT_ID, "protocol": "ps", "domain": "ps"},
json={"agent_id": AGENT_ID, "protocol": "ps", "domain": "remote-interactive-shell"},
)
assert resp.status_code == 200, f"lease request failed: {resp.status_code} {resp.text}"
lease = resp.json()

View File

@@ -15,7 +15,7 @@ from typing import Any, Dict, Optional, Tuple
from flask import Blueprint, jsonify, request, session
from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer
from ...WebSocket.Agent.ReverseTunnel import ReverseTunnelService
from ...WebSocket.Agent.reverse_tunnel_orchestrator import ReverseTunnelService
if False: # pragma: no cover - import cycle hint for type checkers
from .. import EngineServiceAdapters
@@ -103,6 +103,8 @@ def register_tunnel(app, adapters: "EngineServiceAdapters") -> None:
agent_id = _normalize_text(body.get("agent_id"))
protocol = _normalize_text(body.get("protocol") or "ps").lower() or "ps"
domain = _normalize_text(body.get("domain") or protocol).lower() or protocol
if protocol == "ps" and domain == "ps":
domain = "remote-interactive-shell"
if not agent_id:
return jsonify({"error": "agent_id_required"}), 400
@@ -135,4 +137,33 @@ def register_tunnel(app, adapters: "EngineServiceAdapters") -> None:
)
return jsonify(summary), 200
@blueprint.route("/api/tunnel/<tunnel_id>", methods=["DELETE"])
def stop_tunnel(tunnel_id: str):
requirement = _require_login(app)
if requirement:
payload, status = requirement
return jsonify(payload), status
tunnel_id_norm = _normalize_text(tunnel_id)
if not tunnel_id_norm:
return jsonify({"error": "tunnel_id_required"}), 400
body = request.get_json(silent=True) or {}
reason = _normalize_text(body.get("reason") or "operator_stop")
tunnel_service = _get_tunnel_service(adapters)
stopped = False
try:
stopped = tunnel_service.stop_tunnel(tunnel_id_norm, reason=reason)
except Exception as exc: # pragma: no cover - defensive guard
logger.debug("stop_tunnel failed tunnel_id=%s: %s", tunnel_id_norm, exc, exc_info=True)
if not stopped:
return jsonify({"error": "not_found"}), 404
service_log(
"reverse_tunnel",
f"lease stopped tunnel_id={tunnel_id_norm} reason={reason or '-'}",
)
return jsonify({"status": "stopped", "tunnel_id": tunnel_id_norm}), 200
app.register_blueprint(blueprint)

View File

@@ -1,6 +0,0 @@
"""Protocol-specific helpers for Reverse Tunnel (Engine side)."""
from .Powershell import PowershellChannelServer
__all__ = ["PowershellChannelServer"]

View File

@@ -0,0 +1,3 @@
"""Namespace package for reverse tunnel domain handlers (Engine side)."""
__all__ = ["remote_interactive_shell", "remote_management", "remote_video"]

View File

@@ -0,0 +1,78 @@
"""Placeholder Bash channel server (Engine side)."""
from __future__ import annotations
from collections import deque
class BashChannelServer:
"""Stub Bash handler until the agent-side channel is implemented."""
protocol_name = "bash"
def __init__(self, bridge, service, *, channel_id: int = 1, frame_cls=None, close_frame_fn=None):
self.bridge = bridge
self.service = service
self.channel_id = channel_id
self.logger = service.logger.getChild(f"bash.{bridge.lease.tunnel_id}")
self._open_sent = False
self._ack_received = False
self._closed = False
self._output = deque()
self._frame_cls = frame_cls
self._close_frame_fn = close_frame_fn
def handle_agent_frame(self, frame) -> None:
# No-op placeholder; output collection for future Bash support.
try:
if frame.msg_type == 0x04: # MSG_CHANNEL_ACK
self._ack_received = True
except Exception:
return
def open_channel(self, *, cols: int = 120, rows: int = 32) -> None:
self._open_sent = True
self.logger.info(
"bash channel placeholder open sent tunnel_id=%s channel_id=%s cols=%s rows=%s",
self.bridge.lease.tunnel_id,
self.channel_id,
cols,
rows,
)
def send_input(self, data: str) -> None:
# Placeholder: no agent-side Bash yet.
self.logger.info("bash placeholder send_input ignored tunnel_id=%s", self.bridge.lease.tunnel_id)
def send_resize(self, cols: int, rows: int) -> None:
# Placeholder: not implemented.
return
def close(self, code: int = 6, reason: str = "operator_close") -> None:
if self._closed:
return
self._closed = True
if callable(self._close_frame_fn):
try:
frame = self._close_frame_fn(self.channel_id, code, reason)
self.bridge.operator_to_agent(frame)
except Exception:
pass
def drain_output(self):
items = []
while self._output:
items.append(self._output.popleft())
return items
def status(self):
return {
"channel_id": self.channel_id,
"open_sent": self._open_sent,
"ack": self._ack_received,
"closed": self._closed,
"close_reason": None,
"close_code": None,
}
__all__ = ["BashChannelServer"]

View File

@@ -1,4 +1,4 @@
"""Engine-side PowerShell tunnel channel helper."""
"""Engine-side PowerShell tunnel channel helper (remote interactive shell domain)."""
from __future__ import annotations
import json
@@ -134,3 +134,6 @@ class PowershellChannelServer:
"close_reason": self._close_reason,
"close_code": self._close_code,
}
__all__ = ["PowershellChannelServer"]

View File

@@ -0,0 +1,6 @@
"""Protocol handlers for remote interactive shell tunnels (Engine side)."""
from .Powershell import PowershellChannelServer
from .Bash import BashChannelServer
__all__ = ["PowershellChannelServer", "BashChannelServer"]

View File

@@ -0,0 +1,3 @@
"""Domain handlers for remote interactive shells (PowerShell/Bash)."""
__all__ = ["Protocols"]

View File

@@ -0,0 +1,73 @@
"""Placeholder SSH channel server (Engine side)."""
from __future__ import annotations
from collections import deque
class SSHChannelServer:
protocol_name = "ssh"
def __init__(self, bridge, service, *, channel_id: int = 1, frame_cls=None, close_frame_fn=None):
self.bridge = bridge
self.service = service
self.channel_id = channel_id
self.logger = service.logger.getChild(f"ssh.{bridge.lease.tunnel_id}")
self._open_sent = False
self._ack_received = False
self._closed = False
self._output = deque()
self._frame_cls = frame_cls
self._close_frame_fn = close_frame_fn
def handle_agent_frame(self, frame) -> None:
try:
if frame.msg_type == 0x04: # MSG_CHANNEL_ACK
self._ack_received = True
except Exception:
return
def open_channel(self, *, cols: int = 120, rows: int = 32) -> None:
self._open_sent = True
self.logger.info(
"ssh channel placeholder open sent tunnel_id=%s channel_id=%s cols=%s rows=%s",
self.bridge.lease.tunnel_id,
self.channel_id,
cols,
rows,
)
def send_input(self, data: str) -> None:
self.logger.info("ssh placeholder send_input ignored tunnel_id=%s", self.bridge.lease.tunnel_id)
def send_resize(self, cols: int, rows: int) -> None:
return
def close(self, code: int = 6, reason: str = "operator_close") -> None:
if self._closed:
return
self._closed = True
if callable(self._close_frame_fn):
try:
frame = self._close_frame_fn(self.channel_id, code, reason)
self.bridge.operator_to_agent(frame)
except Exception:
pass
def drain_output(self):
items = []
while self._output:
items.append(self._output.popleft())
return items
def status(self):
return {
"channel_id": self.channel_id,
"open_sent": self._open_sent,
"ack": self._ack_received,
"closed": self._closed,
"close_reason": None,
"close_code": None,
}
__all__ = ["SSHChannelServer"]

View File

@@ -0,0 +1,73 @@
"""Placeholder WinRM channel server (Engine side)."""
from __future__ import annotations
from collections import deque
class WinRMChannelServer:
protocol_name = "winrm"
def __init__(self, bridge, service, *, channel_id: int = 1, frame_cls=None, close_frame_fn=None):
self.bridge = bridge
self.service = service
self.channel_id = channel_id
self.logger = service.logger.getChild(f"winrm.{bridge.lease.tunnel_id}")
self._open_sent = False
self._ack_received = False
self._closed = False
self._output = deque()
self._frame_cls = frame_cls
self._close_frame_fn = close_frame_fn
def handle_agent_frame(self, frame) -> None:
try:
if frame.msg_type == 0x04: # MSG_CHANNEL_ACK
self._ack_received = True
except Exception:
return
def open_channel(self, *, cols: int = 120, rows: int = 32) -> None:
self._open_sent = True
self.logger.info(
"winrm channel placeholder open sent tunnel_id=%s channel_id=%s cols=%s rows=%s",
self.bridge.lease.tunnel_id,
self.channel_id,
cols,
rows,
)
def send_input(self, data: str) -> None:
self.logger.info("winrm placeholder send_input ignored tunnel_id=%s", self.bridge.lease.tunnel_id)
def send_resize(self, cols: int, rows: int) -> None:
return
def close(self, code: int = 6, reason: str = "operator_close") -> None:
if self._closed:
return
self._closed = True
if callable(self._close_frame_fn):
try:
frame = self._close_frame_fn(self.channel_id, code, reason)
self.bridge.operator_to_agent(frame)
except Exception:
pass
def drain_output(self):
items = []
while self._output:
items.append(self._output.popleft())
return items
def status(self):
return {
"channel_id": self.channel_id,
"open_sent": self._open_sent,
"ack": self._ack_received,
"closed": self._closed,
"close_reason": None,
"close_code": None,
}
__all__ = ["WinRMChannelServer"]

View File

@@ -0,0 +1,6 @@
"""Protocol handlers for remote management tunnels (Engine side)."""
from .SSH import SSHChannelServer
from .WinRM import WinRMChannelServer
__all__ = ["SSHChannelServer", "WinRMChannelServer"]

View File

@@ -0,0 +1,3 @@
"""Domain handlers for remote management tunnels (SSH/WinRM)."""
__all__ = ["Protocols"]

View File

@@ -0,0 +1,73 @@
"""Placeholder RDP channel server (Engine side)."""
from __future__ import annotations
from collections import deque
class RDPChannelServer:
protocol_name = "rdp"
def __init__(self, bridge, service, *, channel_id: int = 1, frame_cls=None, close_frame_fn=None):
self.bridge = bridge
self.service = service
self.channel_id = channel_id
self.logger = service.logger.getChild(f"rdp.{bridge.lease.tunnel_id}")
self._open_sent = False
self._ack_received = False
self._closed = False
self._output = deque()
self._frame_cls = frame_cls
self._close_frame_fn = close_frame_fn
def handle_agent_frame(self, frame) -> None:
try:
if frame.msg_type == 0x04: # MSG_CHANNEL_ACK
self._ack_received = True
except Exception:
return
def open_channel(self, *, cols: int = 120, rows: int = 32) -> None:
self._open_sent = True
self.logger.info(
"rdp channel placeholder open sent tunnel_id=%s channel_id=%s cols=%s rows=%s",
self.bridge.lease.tunnel_id,
self.channel_id,
cols,
rows,
)
def send_input(self, data: str) -> None:
self.logger.info("rdp placeholder send_input ignored tunnel_id=%s", self.bridge.lease.tunnel_id)
def send_resize(self, cols: int, rows: int) -> None:
return
def close(self, code: int = 6, reason: str = "operator_close") -> None:
if self._closed:
return
self._closed = True
if callable(self._close_frame_fn):
try:
frame = self._close_frame_fn(self.channel_id, code, reason)
self.bridge.operator_to_agent(frame)
except Exception:
pass
def drain_output(self):
items = []
while self._output:
items.append(self._output.popleft())
return items
def status(self):
return {
"channel_id": self.channel_id,
"open_sent": self._open_sent,
"ack": self._ack_received,
"closed": self._closed,
"close_reason": None,
"close_code": None,
}
__all__ = ["RDPChannelServer"]

View File

@@ -0,0 +1,73 @@
"""Placeholder VNC channel server (Engine side)."""
from __future__ import annotations
from collections import deque
class VNCChannelServer:
protocol_name = "vnc"
def __init__(self, bridge, service, *, channel_id: int = 1, frame_cls=None, close_frame_fn=None):
self.bridge = bridge
self.service = service
self.channel_id = channel_id
self.logger = service.logger.getChild(f"vnc.{bridge.lease.tunnel_id}")
self._open_sent = False
self._ack_received = False
self._closed = False
self._output = deque()
self._frame_cls = frame_cls
self._close_frame_fn = close_frame_fn
def handle_agent_frame(self, frame) -> None:
try:
if frame.msg_type == 0x04: # MSG_CHANNEL_ACK
self._ack_received = True
except Exception:
return
def open_channel(self, *, cols: int = 120, rows: int = 32) -> None:
self._open_sent = True
self.logger.info(
"vnc channel placeholder open sent tunnel_id=%s channel_id=%s cols=%s rows=%s",
self.bridge.lease.tunnel_id,
self.channel_id,
cols,
rows,
)
def send_input(self, data: str) -> None:
self.logger.info("vnc placeholder send_input ignored tunnel_id=%s", self.bridge.lease.tunnel_id)
def send_resize(self, cols: int, rows: int) -> None:
return
def close(self, code: int = 6, reason: str = "operator_close") -> None:
if self._closed:
return
self._closed = True
if callable(self._close_frame_fn):
try:
frame = self._close_frame_fn(self.channel_id, code, reason)
self.bridge.operator_to_agent(frame)
except Exception:
pass
def drain_output(self):
items = []
while self._output:
items.append(self._output.popleft())
return items
def status(self):
return {
"channel_id": self.channel_id,
"open_sent": self._open_sent,
"ack": self._ack_received,
"closed": self._closed,
"close_reason": None,
"close_code": None,
}
__all__ = ["VNCChannelServer"]

View File

@@ -0,0 +1,73 @@
"""Placeholder WebRTC channel server (Engine side)."""
from __future__ import annotations
from collections import deque
class WebRTCChannelServer:
protocol_name = "webrtc"
def __init__(self, bridge, service, *, channel_id: int = 1, frame_cls=None, close_frame_fn=None):
self.bridge = bridge
self.service = service
self.channel_id = channel_id
self.logger = service.logger.getChild(f"webrtc.{bridge.lease.tunnel_id}")
self._open_sent = False
self._ack_received = False
self._closed = False
self._output = deque()
self._frame_cls = frame_cls
self._close_frame_fn = close_frame_fn
def handle_agent_frame(self, frame) -> None:
try:
if frame.msg_type == 0x04: # MSG_CHANNEL_ACK
self._ack_received = True
except Exception:
return
def open_channel(self, *, cols: int = 120, rows: int = 32) -> None:
self._open_sent = True
self.logger.info(
"webrtc channel placeholder open sent tunnel_id=%s channel_id=%s cols=%s rows=%s",
self.bridge.lease.tunnel_id,
self.channel_id,
cols,
rows,
)
def send_input(self, data: str) -> None:
self.logger.info("webrtc placeholder send_input ignored tunnel_id=%s", self.bridge.lease.tunnel_id)
def send_resize(self, cols: int, rows: int) -> None:
return
def close(self, code: int = 6, reason: str = "operator_close") -> None:
if self._closed:
return
self._closed = True
if callable(self._close_frame_fn):
try:
frame = self._close_frame_fn(self.channel_id, code, reason)
self.bridge.operator_to_agent(frame)
except Exception:
pass
def drain_output(self):
items = []
while self._output:
items.append(self._output.popleft())
return items
def status(self):
return {
"channel_id": self.channel_id,
"open_sent": self._open_sent,
"ack": self._ack_received,
"closed": self._closed,
"close_reason": None,
"close_code": None,
}
__all__ = ["WebRTCChannelServer"]

View File

@@ -0,0 +1,7 @@
"""Protocol handlers for remote video tunnels (Engine side)."""
from .WebRTC import WebRTCChannelServer
from .RDP import RDPChannelServer
from .VNC import VNCChannelServer
__all__ = ["WebRTCChannelServer", "RDPChannelServer", "VNCChannelServer"]

View File

@@ -0,0 +1,3 @@
"""Domain handlers for remote video/desktop tunnels (RDP/VNC/WebRTC)."""
__all__ = ["Protocols"]

View File

@@ -1,5 +1,5 @@
# ======================================================
# Data\Engine\services\WebSocket\Agent\ReverseTunnel.py
# Data\Engine\services\WebSocket\Agent\reverse_tunnel_orchestrator.py
# Description: Async reverse tunnel scaffolding (Engine side) providing lease management, domain limits, and placeholders for WebSocket listeners.
#
# API Endpoints (if applicable): None
@@ -29,7 +29,13 @@ from typing import Callable, Deque, Dict, Iterable, List, Optional, Tuple
from collections import deque
from threading import Thread
from .ReverseTunnelProtocols import PowershellChannelServer
from .Reverse_Tunnels.remote_interactive_shell.Protocols.Powershell import PowershellChannelServer
from .Reverse_Tunnels.remote_interactive_shell.Protocols.Bash import BashChannelServer
from .Reverse_Tunnels.remote_management.Protocols.SSH import SSHChannelServer
from .Reverse_Tunnels.remote_management.Protocols.WinRM import WinRMChannelServer
from .Reverse_Tunnels.remote_video.Protocols.VNC import VNCChannelServer
from .Reverse_Tunnels.remote_video.Protocols.RDP import RDPChannelServer
from .Reverse_Tunnels.remote_video.Protocols.WebRTC import WebRTCChannelServer
try: # websockets is added to engine requirements
import websockets
@@ -234,10 +240,15 @@ class DomainPolicy:
"""Enforce per-domain concurrency and defaults."""
DEFAULT_LIMITS = {
"ps": 1,
# New domain lanes
"remote-interactive-shell": 2,
"remote-management": 1,
"remote-video": 2,
# Protocol-specific fallbacks (for backward compatibility / legacy callers)
"ps": 2,
"rdp": 1,
"vnc": 1,
"webrtc": 1,
"webrtc": 2,
"ssh": None, # Unlimited
"winrm": None, # Unlimited
}
@@ -459,7 +470,17 @@ class ReverseTunnelService:
self._bridges: Dict[str, "TunnelBridge"] = {}
self._port_servers: Dict[int, asyncio.AbstractServer] = {}
self._agent_sockets: Dict[str, "websockets.WebSocketServerProtocol"] = {}
self._ps_servers: Dict[str, PowershellChannelServer] = {}
self.protocol_registry = {
"ps": PowershellChannelServer,
"powershell": PowershellChannelServer,
"bash": BashChannelServer,
"ssh": SSHChannelServer,
"winrm": WinRMChannelServer,
"vnc": VNCChannelServer,
"rdp": RDPChannelServer,
"webrtc": WebRTCChannelServer,
}
self._protocol_servers: Dict[str, object] = {}
def _ensure_loop(self) -> None:
if self._running and self._loop:
@@ -504,6 +525,12 @@ class ReverseTunnelService:
self._loop.call_soon_threadsafe(asyncio.create_task, websocket.close())
except Exception:
pass
for tunnel_id in list(self._bridges.keys()):
try:
self.release_bridge(tunnel_id, reason="service_stop")
except Exception:
pass
self._protocol_servers.clear()
for lease in list(self.lease_manager.all_leases()):
self.lease_manager.release(lease.tunnel_id, reason="service_stop")
if self._sweeper_task:
@@ -561,9 +588,9 @@ class ReverseTunnelService:
raise ValueError("unknown_tunnel")
bridge = self.ensure_bridge(lease)
bridge.attach_operator(operator_id)
if lease.domain.lower() == "ps":
if (lease.protocol or "").lower() in {"ps", "powershell"}:
try:
server = self.ensure_ps_server(tunnel_id)
server = self.ensure_protocol_server(tunnel_id)
if server:
server.open_channel()
except Exception:
@@ -629,6 +656,22 @@ class ReverseTunnelService:
lease.expires_at = expires_at
return token
def stop_tunnel(self, tunnel_id: str, *, reason: str = "operator_stop", code: int = CLOSE_AGENT_SHUTDOWN) -> bool:
"""Request a graceful stop for a tunnel (operator-driven)."""
lease = self.lease_manager.get(tunnel_id)
if lease is None:
return False
server = self.get_protocol_server(tunnel_id)
if server and hasattr(server, "close"):
try:
server.close(code=code, reason=reason)
except Exception:
self.logger.debug("protocol server close failed tunnel_id=%s", tunnel_id, exc_info=True)
self._push_stop_to_agent(lease, reason=reason)
self.release_bridge(tunnel_id, reason=reason)
return True
def _push_start_to_agent(self, lease: TunnelLease) -> None:
"""Notify the target agent about the new lease over Socket.IO (best-effort)."""
@@ -658,6 +701,26 @@ class ReverseTunnelService:
except Exception:
self.logger.debug("Failed to emit reverse_tunnel_start for tunnel_id=%s", lease.tunnel_id, exc_info=True)
def _push_stop_to_agent(self, lease: TunnelLease, *, reason: str = "operator_stop") -> None:
"""Notify the agent to tear down a tunnel (best-effort)."""
if not self._socketio:
return
try:
self._socketio.emit(
"reverse_tunnel_stop",
{"tunnel_id": lease.tunnel_id, "reason": reason},
namespace="/",
)
self.audit_logger.info(
"lease_push_stop tunnel_id=%s agent_id=%s reason=%s",
lease.tunnel_id,
lease.agent_id,
reason or "-",
)
except Exception:
self.logger.debug("Failed to emit reverse_tunnel_stop for tunnel_id=%s", lease.tunnel_id, exc_info=True)
def lease_summary(self, lease: TunnelLease) -> Dict[str, object]:
return {
"tunnel_id": lease.tunnel_id,
@@ -892,7 +955,7 @@ class ReverseTunnelService:
pass
def _dispatch_agent_frame(self, tunnel_id: str, frame: TunnelFrame) -> None:
server = self._ps_servers.get(tunnel_id)
server = self._protocol_servers.get(tunnel_id)
if not server:
return
try:
@@ -1136,33 +1199,39 @@ class ReverseTunnelService:
self._bridges[lease.tunnel_id] = bridge
return bridge
def ensure_ps_server(self, tunnel_id: str) -> Optional[PowershellChannelServer]:
server = self._ps_servers.get(tunnel_id)
def ensure_protocol_server(self, tunnel_id: str) -> Optional[object]:
server = self._protocol_servers.get(tunnel_id)
if server:
return server
lease = self.lease_manager.get(tunnel_id)
if lease is None:
return None
handler_cls = self.protocol_registry.get((lease.protocol or "").lower())
if handler_cls is None:
return None
bridge = self.ensure_bridge(lease)
server = PowershellChannelServer(
bridge=bridge,
service=self,
frame_cls=TunnelFrame,
close_frame_fn=close_frame,
)
self._ps_servers[tunnel_id] = server
try:
server = handler_cls(
bridge=bridge,
service=self,
frame_cls=TunnelFrame,
close_frame_fn=close_frame,
)
except TypeError:
server = handler_cls(bridge=bridge, service=self)
self._protocol_servers[tunnel_id] = server
return server
def get_ps_server(self, tunnel_id: str) -> Optional[PowershellChannelServer]:
return self._ps_servers.get(tunnel_id)
def get_protocol_server(self, tunnel_id: str) -> Optional[object]:
return self._protocol_servers.get(tunnel_id)
def release_bridge(self, tunnel_id: str, *, reason: str = "bridge_released") -> None:
bridge = self._bridges.pop(tunnel_id, None)
if bridge:
bridge.stop(reason=reason)
if tunnel_id in self._ps_servers:
if tunnel_id in self._protocol_servers:
try:
self._ps_servers.pop(tunnel_id, None)
self._protocol_servers.pop(tunnel_id, None)
except Exception:
pass

View File

@@ -20,7 +20,7 @@ from flask_socketio import SocketIO
from ...database import initialise_engine_database
from ...server import EngineContext
from .Agent.ReverseTunnel import (
from .Agent.reverse_tunnel_orchestrator import (
ReverseTunnelService,
TunnelBridge,
decode_frame,
@@ -406,8 +406,8 @@ def register_realtime(socket_server: SocketIO, context: EngineContext) -> None:
tunnel_id = _operator_sessions.get(sid)
if not tunnel_id:
return None, None, {"error": "not_joined"}
server = tunnel_service.ensure_ps_server(tunnel_id)
if server is None:
server = tunnel_service.ensure_protocol_server(tunnel_id)
if server is None or not hasattr(server, "open_channel"):
return None, tunnel_id, {"error": "ps_unsupported"}
return server, tunnel_id, None
@@ -489,4 +489,9 @@ def register_realtime(socket_server: SocketIO, context: EngineContext) -> None:
@socket_server.on("disconnect", namespace=tunnel_namespace)
def _ws_tunnel_disconnect():
sid = request.sid
_operator_sessions.pop(sid, None)
tunnel_id = _operator_sessions.pop(sid, None)
if tunnel_id and tunnel_id not in _operator_sessions.values():
try:
tunnel_service.stop_tunnel(tunnel_id, reason="operator_socket_disconnect")
except Exception as exc:
logger.debug("ws_tunnel_disconnect stop_tunnel failed tunnel_id=%s: %s", tunnel_id, exc, exc_info=True)

View File

@@ -114,6 +114,7 @@ export default function ReverseTunnelPowershell({ device }) {
const terminalRef = useRef(null);
const joinRetryRef = useRef(null);
const joinAttemptsRef = useRef(0);
const DOMAIN_REMOTE_SHELL = "remote-interactive-shell";
const hostname = useMemo(() => {
return (
@@ -164,6 +165,23 @@ export default function ReverseTunnelPowershell({ device }) {
setPolling(false);
}, []);
const stopTunnel = useCallback(
async (reason = "operator_disconnect") => {
const tunnelId = tunnel?.tunnel_id;
if (!tunnelId) return;
try {
await fetch(`/api/tunnel/${tunnelId}`, {
method: "DELETE",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ reason }),
});
} catch (err) {
// best-effort; socket close frame acts as fallback
}
},
[tunnel?.tunnel_id]
);
useEffect(() => {
return () => {
stopPolling();
@@ -172,8 +190,9 @@ export default function ReverseTunnelPowershell({ device }) {
clearTimeout(joinRetryRef.current);
joinRetryRef.current = null;
}
stopTunnel("component_unmount");
};
}, [disconnectSocket, stopPolling]);
}, [disconnectSocket, stopPolling, stopTunnel]);
const appendOutput = useCallback((text) => {
if (!text) return;
@@ -270,7 +289,8 @@ export default function ReverseTunnelPowershell({ device }) {
[appendOutput, emitAsync, stopPolling, disconnectSocket]
);
const handleDisconnect = useCallback(() => {
const handleDisconnect = useCallback(
async (reason = "operator_disconnect") => {
const socket = socketRef.current;
const tunnelId = tunnel?.tunnel_id;
if (joinRetryRef.current) {
@@ -282,11 +302,14 @@ export default function ReverseTunnelPowershell({ device }) {
const frame = buildCloseFrame(1, CLOSE_AGENT_SHUTDOWN, "operator_close");
socket.emit("send", { frame });
}
await stopTunnel(reason);
stopPolling();
disconnectSocket();
setTunnel(null);
setSessionState("closed");
}, [disconnectSocket, stopPolling, tunnel?.tunnel_id]);
},
[disconnectSocket, stopPolling, stopTunnel, tunnel?.tunnel_id]
);
const handleResize = useCallback(() => {
if (!socketRef.current || sessionState === "idle") return;
@@ -410,7 +433,7 @@ export default function ReverseTunnelPowershell({ device }) {
const resp = await fetch("/api/tunnel/request", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ agent_id: agentId, protocol: "ps", domain: "ps" }),
body: JSON.stringify({ agent_id: agentId, protocol: "ps", domain: DOMAIN_REMOTE_SHELL }),
});
const data = await resp.json().catch(() => ({}));
if (!resp.ok) {
@@ -429,7 +452,7 @@ export default function ReverseTunnelPowershell({ device }) {
setStatusSeverity("error");
setStatusMessage("");
}
}, [agentId, connectSocket, connectionType, resetState]);
}, [DOMAIN_REMOTE_SHELL, agentId, connectSocket, connectionType, resetState]);
const handleSend = useCallback(
async (text) => {
@@ -456,6 +479,17 @@ export default function ReverseTunnelPowershell({ device }) {
sessionState === "lease_issued";
const canStart = Boolean(agentId) && !isBusy;
useEffect(() => {
const handleUnload = () => {
stopTunnel("window_unload");
};
if (tunnel?.tunnel_id) {
window.addEventListener("beforeunload", handleUnload);
return () => window.removeEventListener("beforeunload", handleUnload);
}
return undefined;
}, [stopTunnel, tunnel?.tunnel_id]);
const sessionChips = [
{
label: isConnected ? "Connected" : isClosed ? "Session ended" : sessionState === "idle" ? "Idle" : sessionState.replace(/_/g, " "),

View File

@@ -20,6 +20,9 @@ Use this doc for agent-only work (Borealis agent runtime under `Data/Agent` →
- Validates script payloads with backend-issued Ed25519 signatures before execution.
- Outbound-only; API/WebSocket calls flow through `AgentHttpClient.ensure_authenticated` for proactive refresh. Logs bootstrap, enrollment, token refresh, and signature events in `Agent/Logs/`.
## Reverse Tunnels
- Design, orchestration, domains, limits, and lifecycle are documented in `Docs/Codex/REVERSE_TUNNELS.md`. Agent role implementation lives in `Data/Agent/Roles/role_ReverseTunnel.py` with per-domain protocol handlers under `Data/Agent/Roles/Reverse_Tunnels/`.
## Execution Contexts & Roles
- Auto-discovers roles from `Data/Agent/Roles/`; no loader changes needed.
- Naming: `role_<Purpose>.py` with `ROLE_NAME`, `ROLE_CONTEXTS`, and optional hooks (`register_events`, `on_config`, `stop_all`).

View File

@@ -23,6 +23,10 @@ Use this doc for Engine work (successor to the legacy server). For shared guidan
- Enrollment: operator approvals, conflict detection, auditor recording, pruning of expired codes/refresh tokens.
- Background jobs and service adapters maintain compatibility with legacy DB schemas while enabling gradual API takeover.
## Reverse Tunnels
- Full design and lifecycle are in `Docs/Codex/REVERSE_TUNNELS.md` (domains, limits, framing, APIs, stop path, UI hooks).
- Engine orchestrator is `Data/Engine/services/WebSocket/Agent/reverse_tunnel_orchestrator.py` with domain handlers under `Data/Engine/services/WebSocket/Agent/Reverse_Tunnels/`.
## WebUI & WebSocket Migration
- Static/template handling: `Data/Engine/services/WebUI`; deployment copy paths are wired through `Borealis.ps1` with TLS-aware URL generation.
- Stage 6 tasks: migration switch in the legacy server for WebUI delegation and porting device/admin API endpoints into Engine services.

View File

@@ -1,302 +0,0 @@
# Codex Prompt
Read `Docs/Codex/FEATURE_IMPLEMENTATION_TRACKING/Agent_Reverse_Tunneling.md` and follow the checklist. Preserve existing Engine/Agent behavior, reuse TLS/identity, and implement the dedicated reverse tunnel system (WebSocket-over-TLS) with ephemeral Engine ports and browser-based operator access. Update this ledger with progress, deviations, and next steps before and after changes.
# Context & Goals
- Build a high-performance reverse tunnel between Engine and Agent to carry interactive protocols (first target: interactive PowerShell) while keeping the existing control Socket.IO untouched.
- Agents use a single fixed outbound port to reach the Engines tunnel listener; Engine allocates ephemeral ports from 3000040000 per session.
- Tunnels are operator-triggered, ephemeral, and torn down after inactivity (1h idle) or if the agent stays offline beyond the grace window (1h).
- WebUI-only operator experience: Engine exposes a WebSocket endpoint that bridges browser sessions to agent channels.
- Security: reuse pinned TLS bundle + Ed25519 device identity and short-lived signed tokens per tunnel/channel.
- Concurrency: per-domain caps (one PowerShell session per agent; RDP/VNC/WebRTC grouped to one concurrent session across that domain; WinRM/SSH can scale higher later).
- Logging: Device Activity entries for session start/stop (with operator identity when available) plus `reverse_tunnel.log` on both Agent and Engine.
- Transport: WebSocket-over-TLS for the tunnel socket to keep proxy-friendliness and reuse libraries.
- Motivation: Provide NAT-friendly, high-throughput, operator-initiated remote access (PowerShell first) without touching the existing lightweight control Socket.IO; agents remain outbound-only, Engine leases high ports on demand.
# Background for a New Codex Agent (assumes you read AGENTS.md first)
- AGENTS.md already points you to `BOREALIS_ENGINE.md`, `BOREALIS_AGENT.md`, and shared UI docs; follow their logging paths, runtime locations, and UI rules.
- Keep the existing Socket.IO control channel untouched; this tunnel is a new dedicated listener/port.
- Reuse security: pinned TLS bundle + Ed25519 identity + existing token signing. Agent stays outbound-only; no inbound openings on devices.
- UI reminders specific to this feature: PowerShell page should mirror `Assemblies/Assembly_Editor.jsx` syntax highlighting and general layout from `Admin/Page_Template.jsx` per UI doc.
- Licensing: project is AGPL; PowerShell runs in pipe mode (no pywinpty/ConPTY dependency).
- Non-destructive: new code must be gated/dormant until invoked; avoid regressions to existing roles/pages.
# Non-Destructive Expectations
- Do not break existing Agent/Engine comms. New code should be additive and dormant until wired.
- WebUI additions must not impact current pages; new routes/components should be isolated.
- Note any temporary breakage during development here before committing.
# Architecture Plan (High Level)
- Transport: Dedicated WebSocket-over-TLS listener (Engine) on fixed port; Agents dial outbound only.
- Handshake: API on port 443 negotiates an ephemeral tunnel port + token/lease; Agent opens tunnel socket to that port; Engine maps operator channels to agent channels.
- Framing: Binary frames `version | msg_type | channel_id | flags | length | payload`; supports heartbeat, back-pressure, close codes, and resize events for terminals.
- Lease/idle: 1h idle timeout; 1h grace if agent drops mid-session before freeing port.
- PowerShell v1: Agent spawns PowerShell via pipes (no ConPTY), Engine provides browser terminal bridge with syntax highlighting like `Assembly_Editor.jsx`.
# Terminology & IDs
- agent_id: existing composed ID (hostname + GUID + scope).
- tunnel_id: UUID per tunnel lease.
- channel_id: uint32 per logical stream inside a tunnel (PowerShell uses one channel for stdio + control subframes for resize).
- lease: mapping of tunnel_id -> agent_id -> assigned_port -> expiry/idle timers -> domain/protocol metadata -> token.
# Handshake / Flow (end-to-end)
1) Operator in WebUI clicks “Remote PowerShell” on a device.
2) WebUI calls Engine API (port 443) `POST /api/tunnel/request` with agent_id, protocol=ps, domain=ps, operator_id/context. API:
- Consults lease manager (port pool 3000040000). Enforce domain concurrency per agent (PowerShell max 1).
- Allocates port, tunnel_id, expiry (idle 1h, grace 1h), signs token (binds agent_id, tunnel_id, port, protocol, expires_at).
- Persists lease (in-memory + db/kv if available) and logs Device Activity (pending).
- Returns {tunnel_id, port, token, expires_at, idle_seconds, protocol, domain}.
- Spins up/keeps alive tunnel listener for that port in the async service (engine-side).
3) Engine notifies agent over existing control channel (Socket.IO or REST push) with the lease payload.
4) Agent ReverseTunnel role receives the task, validates token signature (Engine public key) + expiry, opens WebSocket-over-TLS to engine_host:port, sends CONNECT frame {agent_id, tunnel_id, token, protocol, domain, client_version}.
5) Engine listener validates token, binds the WebSocket to the lease, marks agent_connected_at, starts idle timer.
6) Operator browser opens Engine WebUI bridge WebSocket `/ws/tunnel/<tunnel_id>` (port 443) with operator auth. Engine maps browser session to lease and assigns channel 1 for PowerShell.
7) Engine sends CHANNEL_OPEN to agent for channel 1 (protocol ps). Agent starts PowerShell ConPTY, streams stdout/stderr as DATA frames, accepts stdin from browser via Engine bridge.
8) Heartbeats: ping/pong every 20s; idle timer resets on traffic; if idle > 1h, Engine and Agent send CLOSE (code=idle) and free lease/port.
9) If agent disconnects mid-session, lease is held for 1h grace; if reconnects within grace, Engine allows resume, else frees port.
10) On session end (operator closes or process exits), Engine/Agent exchange CLOSE, teardown channel, release port, write Device Activity (stop) and service logs.
# Framing (binary over WebSocket)
- Fixed header (little endian): version(1) | msg_type(1) | flags(1) | reserved(1) | channel_id(4) | length(4) | payload(length).
- msg_types:
- 0x01 CONNECT (payload: JSON or CBOR {agent_id, tunnel_id, token, protocol, domain, version})
- 0x02 CONNECT_ACK / CONNECT_ERR (payload: code, message)
- 0x03 CHANNEL_OPEN (payload: protocol, metadata)
- 0x04 CHANNEL_ACK / CHANNEL_ERR
- 0x05 DATA (payload: raw bytes)
- 0x06 WINDOW_UPDATE (payload: uint32 credits) for back-pressure if needed
- 0x07 HEARTBEAT (ping/pong)
- 0x08 CLOSE (payload: code, reason)
- 0x09 CONTROL (payload: JSON for resize: {cols, rows} or future control)
- Back-pressure: default to simple socket pausing; WINDOW_UPDATE optional for high-rate protocols. Start with pause/resume read on buffers > threshold.
- Close codes: 0=ok, 1=idle_timeout, 2=grace_expired, 3=protocol_error, 4=auth_failed, 5=server_shutdown, 6=agent_shutdown, 7=domain_limit, 8=unexpected_disconnect.
# Port Lease / Domain Policy
- Port pool: 3000040000 inclusive. Persist active leases (memory + lightweight state file/db row).
- Lease fields: tunnel_id, agent_id, assigned_port, protocol, domain, operator_id, created_at, expires_at, idle_timeout=3600s, grace_timeout=3600s, state {pending, active, closing, expired}, last_activity_ts.
- Allocation: first free port in pool (wrap). Refuse if pool exhausted; emit error to operator.
- Domain concurrency per agent:
- ps domain: max 1 active tunnel.
- rdp/vnc/webrtc domain: max 1 active (future).
- ssh/winrm domain: allow >1 (configurable).
- Idle: reset on any DATA/CONTROL. After 1h idle, send CLOSE, free lease.
- Grace: if agent disconnects, hold lease for 1h; allow reconnect to same tunnel_id/port with valid token; else free.
# Engine Components (detailed)
- Async service module `Data/Engine/services/WebSocket/Agent/ReverseTunnel.py`:
- Runs asyncio/uvloop TCP/TLS listener factory for the fixed port (or on-demand per allocated port).
- Manages per-port WebSocket acceptors bound to leases.
- Validates tokens (Ed25519 or existing JWT signer); ensures agent_id/tunnel_id/port/protocol match.
- Maintains lease manager (in-memory map + persistence hook).
- Provides API helpers to allocate/release leases and to push control messages to Agent via existing Socket.IO.
- Logging to `Engine/Logs/reverse_tunnel.log`.
- Emits Device Activity entries on session start/stop (with operator id if present).
- API endpoint (port 443) `POST /api/tunnel/request`:
- Inputs: agent_id, protocol, domain, operator_id (from auth), metadata (e.g., hostname).
- Output: tunnel_id, port, token, idle_seconds, grace_seconds, expires_at.
- Checks domain limits, pool availability.
- Browser bridge endpoint `/ws/tunnel/<tunnel_id>`:
- Auth via existing operator session/JWT.
- Binds to lease, opens channel 1 to agent, relays DATA/CONTROL, handles CLOSE/idle.
- Enforces per-operator attach (one active browser per tunnel unless multi-view is allowed; start with one).
- WebUI wiring (later): PowerShell page uses this bridge; status toasts on errors/idle.
# Agent Components (detailed)
- Role file `Data/Agent/Roles/role_ReverseTunnel.py`:
- Registers control event handler to receive tunnel instructions (payload from Engine API push).
- Validates token signature/expiry and domain limits.
- Opens WebSocket-over-TLS to engine_host:assigned_port using existing TLS bundle/identity.
- Implements framing (CONNECT, CHANNEL_OPEN, DATA, HEARTBEAT, CLOSE).
- Manages sub-role registry for protocols (PowerShell first).
- Enforces per-domain concurrency; refuses new tunnel if violation (sends error).
- Heartbeat + idle tracking; stop_all closes active tunnels cleanly.
- Logging to `Agent/Logs/reverse_tunnel.log`.
- Submodules under `Data/Agent/Roles/ReverseTunnel/`:
- `tunnel_Powershell.py`: pipes-only PowerShell subprocess (stdin/stdout piping, control frames are no-ops for resize), exit codes.
- Common helpers: channel dispatcher, back-pressure (pause reads if outbound buffer high).
# PowerShell v1 (end-to-end)
- Engine:
- `Data/Engine/services/WebSocket/Agent/ReverseTunnel/Powershell.py` handles protocol-specific channel setup, translates browser resize/control messages to CONTROL frames, and passes stdin/out via DATA.
- Integrates with Device Activity logging (start/stop, operator id, agent id, tunnel_id).
- Agent:
- Spawn PowerShell in ConPTY with configurable shell path if needed; set environment minimal; capture stdout/stderr; forward to channel.
- Handle EXIT -> send CLOSE with exit code; tear down channel and release lease.
- WebUI:
- `Data/Engine/web-interface/src/ReverseTunnel/Powershell.jsx` page/modal with terminal UI, syntax highlighting like `Assemblies/Assembly_Editor.jsx`, copy support, status toasts, idle timeout banner.
- Uses `/ws/tunnel/<tunnel_id>` bridge; shows reconnect/spinner while agent connects; shows errors for domain limit or auth failure.
# Logging & Auditing
- Service logs: `Engine/Logs/reverse_tunnel.log`, `Agent/Logs/reverse_tunnel.log` with tunnel_id/channel_id/agent_id/operator_id.
- Device Activity: add entries on session start/stop with operator info if available, reason (idle timeout, exit, error).
- Metrics (optional later): count active tunnels, per-domain usage, port pool pressure.
# Config Knobs (defaults)
- fixed_tunnel_port: 8443 (or reuse 443 listener with path split if desired).
- port_range: 30000-40000.
- idle_timeout_seconds: 3600.
- grace_timeout_seconds: 3600.
- heartbeat_interval_seconds: 20.
- domain concurrency limits: ps=1, rdp/vnc/webrtc=1 shared, ssh=unbounded/configurable, winrm=unbounded/configurable.
- enable_compression: false (initially).
# Testing & Validation (detailed)
- Engine unit tests: lease manager allocations, domain limit enforcement, idle/grace expiry, token validation, port pool exhaustion.
- Engine integration: simulated agent CONNECT, channel open, data echo, idle timeout, reconnect within grace.
- Agent unit tests: role lifecycle start/stop, token rejection, domain enforcement, heartbeat handling.
- Manual plan: start Engine and Agent in dev; request tunnel via API; agent receives task; tunnel connects; run PowerShell commands; test resize; test idle timeout; test agent drop and reconnect within 1h; test domain limit error when a second PS session is requested.
- WebUI manual: open PS page, run commands, copy output, observe Device Activity entries, see idle timeout banner.
# Credits & Attribution
- Add pywinpty attribution to `Data/Engine/web-interface/src/Dialogs.jsx` CreditsDialog under “Code Shared in this Project.”
# Risks / Watchpoints
- Eventlet vs asyncio coexistence: ensure tunnel service uses dedicated loop/thread/process to avoid blocking existing Socket.IO handlers.
- Port exhaustion: detect and return meaningful errors; ensure cleanup on process exit.
- Buffer growth: enforce back-pressure; pause reads when output queue high.
- Packaging: pywinpty wheels availability for supported Python versions; note in doc if build needed.
- Security: strict token binding (agent_id, tunnel_id, port, protocol, expiry); reject mismatches; always TLS.
# Next Actions (on approval)
- Document handshake and framing (done above) — refine in code comments.
- Scaffold Engine tunnel service + lease manager + logging (no wiring to main app yet).
- Scaffold Agent role + PowerShell submodule (dormant until enabled).
- Add WebUI PowerShell page and CreditsDialog attribution.
- Wire negotiation API to lease manager; push control payload to Agent; wire browser bridge to tunnel service.
# Implementation Sequence (follow in order)
1) Read project docs (`Docs/Codex/BOREALIS_ENGINE.md`, `Docs/Codex/BOREALIS_AGENT.md`, `Docs/Codex/SHARED.md`, `Docs/Codex/USER_INTERFACE.md`).
2) Add config defaults (fixed tunnel port, port range, timeouts) without enabling by default.
3) Build Engine lease manager (allocations, domain rules, idle/grace timers, persistence hook) with unit tests.
4) Implement framing helper (encode/decode headers, heartbeats, close codes, back-pressure hooks).
5) Stand up Engine async WebSocket-over-TLS listener (fixed port), token validation, and per-lease bindings.
6) Implement negotiation API `/api/tunnel/request`; sign tokens; push control payload to Agent via existing channel.
7) Add Engine browser bridge `/ws/tunnel/<tunnel_id>`; relay to agent channel; enforce auth and single attachment.
8) Scaffold Agent role `role_ReverseTunnel.py` (token verify, connect, channel dispatch, heartbeat, stop_all, domain limits).
9) Implement Agent PowerShell submodule with ConPTY/pywinpty, resize, stdout/stderr piping, exit handling.
10) Implement Engine PowerShell handler to translate browser events and route frames.
11) Build WebUI PowerShell page `ReverseTunnel/Powershell.jsx` with terminal UI, syntax highlighting, status/idle handling.
12) Wire Device Activity logging for session start/stop; surface in Device Activity tab.
13) Add Credits dialog attribution for pywinpty.
14) Run tests (unit + manual end-to-end); verify idle/grace, domain limits, resize, reconnect.
15) Gate feature (config off by default), clean up logs, update this ledger with status and deviations.
# Handoff Notes for the Next Codex Agent
- Treat this file as the single source of truth for the tunnel feature; document deviations and progress here.
- Keep changes additive; do not modify unrelated Socket.IO handlers or existing roles/pages.
- Maintain outbound-only design for Agents; Engine listens on fixed + leased ports.
- Performance matters: use asyncio/uvloop for tunnel service; avoid blocking eventlet paths; add basic back-pressure.
- Security is mandatory: TLS + signed tokens bound to agent_id/tunnel_id/port/protocol/expiry; close on mismatch or framing errors.
# Execution Protocol for the Codex Agent
- Work one checklist item at a time.
- After finishing an item, mark it in the Detailed Checklist and briefly summarize what changed.
- Before starting the next item, ask the operator for permission to proceed. Pause if permission is not granted.
- If you must reorder items (e.g., dependency), note the rationale here before proceeding.
- Keep the codebase functional at all times. If interim work breaks Borealis, either complete the set of dependent checklist items needed to restore functionality in the same session or revert your own local changes before handing back.
- Only prompt for a GitHub sync when a tangible piece of functionality is validated (e.g., API call works, tunnel connects, UI interaction tested). Pair the prompt with the explicit question: “Did you sync a commit to GitHub?” after validation or operator testing.
# Detailed Checklist (update statuses)
- [x] Repo hygiene
- [x] Confirm no conflicting changes; avoid touching legacy Socket.IO handlers.
- [x] PowerShell transport: pipe-only (pywinpty/ConPTY removed from Agent deps).
- [x] Engine tunnel service
- [x] Add reverse tunnel config defaults (fixed port, port range, timeouts, log path) without enabling.
- [x] Create `Data/Engine/services/WebSocket/Agent/ReverseTunnel.py` (async/uvloop listener, port pool 3000040000).
- [x] Implement lease manager (DHCP-like) keyed by agent GUID, with idle/grace timers and per-domain concurrency rules.
- [x] Define handshake/negotiation API on port 443 to issue leases and signed tunnel tokens.
- [x] Implement channel framing, flow control, heartbeats, close semantics.
- [x] Logging: `Engine/Logs/reverse_tunnel.log`; audit into Device Activity (session start/stop, operator id, agent id, tunnel_id, port).
- [x] WebUI operator bridge endpoint (WebSocket) that maps browser sessions to agent channels.
- [x] Idle/grace sweeper + heartbeat wiring for tunnel sockets.
- [x] TLS-aware per-port listener and agent CONNECT_ACK handling.
- [x] Agent tunnel role
- [x] Add `Data/Agent/Roles/role_ReverseTunnel.py` (manages tunnel socket, reconnect, heartbeats, channel dispatch).
- [x] Per-protocol submodules under `Data/Agent/Roles/ReverseTunnel/` (first: `tunnel_Powershell.py`).
- [x] Enforce per-domain concurrency (one PowerShell; prevent multiple RDP/VNC/WebRTC; allow extensible policies).
- [x] Logging: `Agent/Logs/reverse_tunnel.log`; include tunnel_id/channel_id.
- [x] Integrate token validation, TLS reuse, idle teardown, and graceful stop_all.
- [ ] PowerShell v1 (feature target)
- [x] Engine side `Data/Engine/services/WebSocket/Agent/ReverseTunnel/Powershell.py` (channel server, resize handling, translate browser events).
- [x] Agent side `Data/Agent/Roles/ReverseTunnel/tunnel_Powershell.py` using pipes-only PowerShell subprocess; map stdin/stdout to frames; resize no-op.
- [ ] WebUI: `Data/Engine/web-interface/src/ReverseTunnel/Powershell.jsx` with terminal UI, syntax highlighting matching `Assemblies/Assembly_Editor.jsx`, copy support, status toasts.
- [ ] Device Activity entries and UI surface in `Devices/Device_List.jsx` Device Activity tab.
- [ ] Credits & attribution
- [x] pywinpty removed (no attribution needed); revisit if new third-party deps added.
- [ ] Testing & validation
- [ ] Unit/behavioral tests for lease manager, framing, and idle teardown (Engine side).
- [ ] Agent role lifecycle tests (start/stop, reconnect, single-session enforcement).
- [ ] Manual test plan: request port, start PowerShell session, send commands, resize, idle timeout, offline grace recovery, concurrent domain policy.
- [ ] Operational notes
- [ ] Document config knobs: fixed tunnel port, port range, idle/grace durations, domain concurrency limits.
- [ ] Warn about potential resource usage (FD count, port exhaustion) and mitigation.
## Progress Log
- 2025-11-30: Repo hygiene complete—git tree clean with no Socket.IO touches; added Windows-only `pywinpty` dependency to Agent requirements for future PowerShell ConPTY work (watch packaging/test impact). Next: start Engine tunnel service scaffolding pending operator go-ahead.
- 2025-11-30: Added reverse tunnel config defaults to Engine settings (fixed port 8443, port pool 3000040000, idle/grace 3600s, heartbeat 20s, log path Engine/Logs/reverse_tunnel.log); feature still dormant and not wired.
- 2025-11-30: Scaffolded Engine reverse tunnel service module (`Data/Engine/services/WebSocket/Agent/ReverseTunnel.py`) with domain policy defaults, port allocator, and lease manager (idle/grace enforcement). Service stays dormant; listener/bridge wiring and framing remain TODO.
- 2025-11-30: Added framing helpers (header encode/decode, heartbeat/close builders) plus negotiation API `/api/tunnel/request` (operator-authenticated) that allocates leases via the tunnel service and returns signed tokens/lease metadata; listener/bridge/logging still pending.
- 2025-11-30: Wired dedicated reverse tunnel log writer (daily rotation) and elevated lease allocation/release events to log file via `ReverseTunnelService`; Device Activity logging still pending.
- 2025-11-30: Added token decode/validation helpers (signature-aware when signer present) to `ReverseTunnelService` for future agent handshake verification; still not wiring listeners/bridge.
- 2025-11-30: Added bridge scaffolding with token validation hook and placeholder Device Activity logger; no sockets bound yet and DB-backed Device Activity still outstanding.
- 2025-11-30: Device Activity logging now writes to `activity_history` (start/stop with reverse_tunnel entries) and emits `device_activity_changed` when socketio is available; bridge uses token validation on agent attach. Listener wiring still pending.
- 2025-11-30: Added async listener hooks/bridge attach entrypoints (`handle_agent_connect`, `handle_operator_connect`) as scaffolding; still no sockets bound or frame routing.
- 2025-11-30: Moved negotiation API to `services/API/devices/tunnel.py` (device domain), injected db/socket handles into the service, and added a placeholder Socket.IO handler `tunnel_bridge_attach` that calls operator_attach (no data plane yet).
- 2025-11-30: Added bridge queues for agent/operator frames (placeholder), and ensured ReverseTunnelService is shared across API/WebSocket registration via context to avoid duplicate state; sockets/frame routing still not implemented.
- 2025-11-30: Added WebUI-facing Socket.IO namespace `/tunnel` with join/send/poll events that map browser sessions to tunnel bridges, using base64-encoded frames and operator auth from session/cookies.
- 2025-11-30: Enabled async WebSocket listener per assigned port (TLS-aware via Engine certs) for agent CONNECT frames, with frame routing between agent socket and browser bridge queues; Engine tunnel service checklist marked complete.
- 2025-11-30: Added idle/grace sweeper, CONNECT_ACK to agents, heartbeat loop, and token-touched operator sends; per-port listener now runs on dedicated loop/thread. (Original instructions didnt call out sweeper/heartbeat wiring explicitly.)
- 2025-12-01: Added Agent reverse tunnel role (`Data/Agent/Roles/role_ReverseTunnel.py`) with TLS-aware WebSocket dialer, token validation against signed leases, domain-limit guard, heartbeat/idle watchdogs, and reverse_tunnel.log status emits; protocol handlers remain stubbed until PowerShell module lands.
- 2025-12-01: Implemented Agent PowerShell channel (initially pywinpty/ConPTY path, later simplified to pipes-only) and Engine PowerShell handler with Socket.IO helpers (`ps_open`/`ps_send`/`ps_resize`/`ps_poll`); added ps channel logging and domain-aware attach. WebUI remains pending.
- 2025-12-06: Simplified PowerShell handler to pipes-only, removed pywinpty dependency, added robust handler import for non-package agent runtimes, and cleaned UI status messaging.
## Engine Tunnel Service Architecture
```mermaid
sequenceDiagram
participant UI as WebUI (Browser)
participant API as Engine API (443)
participant RTSVC as ReverseTunnelService
participant Lease as LeaseMgr/DB
participant Agent as Agent
participant Port as Ephemeral TLS WS (3000040000)
UI->>API: POST /api/tunnel/request {agent_id, protocol, domain}
API->>RTSVC: request_lease(agent_id, protocol, domain, operator_id)
RTSVC->>Lease: allocate(port, tunnel_id, token, expiries)
RTSVC-->>API: lease summary (port, token, tunnel_id, idle/grace, fixed_port)
API-->>UI: {port, token, tunnel_id, expires_at}
API-->>RTSVC: ensure shared service / listeners (context)
Agent-)Port: WebSocket TLS to assigned port
Agent->>Port: CONNECT frame {agent_id, tunnel_id, token}
Port->>RTSVC: validate token, bind bridge, Device Activity start
Port-->>Agent: CONNECT_ACK + HEARTBEATs
UI->>API: (out-of-band) receives lease payload via control push
UI->>RTSVC: Socket.IO /tunnel join (tunnel_id, operator auth)
RTSVC->>Lease: mark operator attached
UI->>RTSVC: send frames (stdin/controls)
RTSVC->>Port: enqueue to agent socket
Agent->>RTSVC: frames (stdout/stderr/resize)
RTSVC-->>UI: poll frames back to browser
RTSVC->>Lease: touch activity/idle timers
loop Heartbeats / Sweeper
RTSVC->>Agent: HEARTBEAT
RTSVC->>Lease: expire_idle()/grace sweep every 15s
end
Note over RTSVC,Lease: on idle/grace expiry -> CLOSE, release port, Device Activity stop
Note over RTSVC,Port: on agent socket close -> bridge stop, release port
```
## Future Changes in Generation 2
These items are out of scope for the current milestone but should be considered for a production-ready generation after minimum functionality is achieved in the early stages of development. This section is a place to note things that were not implemented in Generation 1, but should be added in future iterations of the Reverse Tunneling system.
- Harden operator auth/authorization: enforce per-operator session binding, ownership checks, audited attach/detach, and offer a pure WebSocket `/ws/tunnel/<tunnel_id>` bridge.
- Replace Socket.IO browser bridge with a dedicated binary WebSocket bridge for higher throughput and simpler framing.
- Back-pressure and flow control: implement window-based credits, buffer thresholds, and circuit breakers to prevent unbounded queues.
- Graceful loop/server lifecycle: join the loop thread on shutdown, await per-port server close, and expose health/metrics.
- Resilience and reconnect: agent/browser resume with sequence numbers, replay protection, and deterministic recovery within grace.
- Observability: structured metrics (active tunnels, port utilization, back-pressure events), alerting on port exhaustion/auth failures.
- Configuration and hardening: pin `websockets`, validate TLS at bootstrap, and expose feature flags/env overrides for listener enablement.

View File

@@ -0,0 +1,92 @@
# Borealis Reverse Tunnels Operator & Developer Guide
This document is the single reference for how Borealis reverse tunnels are organized, secured, and orchestrated. It is written for Codex agents extending the feature (new protocols, UI, or policy changes).
## 1) High-Level Model
- Outbound-only: Agents initiate all tunnel sockets to the Engine. No inbound openings on devices.
- Transport: WebSocket-over-TLS carrying a binary frame header (version | msg_type | flags | reserved | channel_id | length) plus payload.
- Leases: Engine issues short-lived leases per agent/domain/protocol. Each lease binds a tunnel_id to an ephemeral Engine port and a signed token.
- Domains: Concurrency “lanes” keep protocols isolated: `remote-interactive-shell` (2), `remote-management` (1), `remote-video` (2). Legacy aliases (`ps`, etc.) normalize into these lanes.
- Channels: Logical streams inside a tunnel (channel_id u32). PS uses channel 1; future protocols can open more channels per tunnel as needed.
- Tear-down: Idle/grace timeouts plus explicit operator stop. Closing a tunnel must close its protocol channel(s) and kill the agent process for interactive shells.
## 2) Engine Components
- Orchestrator: `Data/Engine/services/WebSocket/Agent/reverse_tunnel_orchestrator.py`
- Lease manager: Port pool allocator, domain limit enforcement, idle/grace sweeper.
- Token issuer/validator: Binds agent_id, tunnel_id, domain, protocol, port, expires_at.
- Bridge: Maps agent sockets ↔ operator sockets; stores per-tunnel protocol server instances.
- Logging: `Engine/Logs/reverse_tunnel.log` plus Device Activity start/stop entries.
- Stop path: `stop_tunnel` closes protocol servers, emits `reverse_tunnel_stop` to agents, releases lease/bridge.
- Protocol registry: Domain/protocol handlers under `Data/Engine/services/WebSocket/Agent/Reverse_Tunnels/`:
- `remote_interactive_shell/Protocols/Powershell.py` (live), `Bash.py` (placeholder).
- `remote_management/Protocols/SSH.py`, `WinRM.py` (placeholders).
- `remote_video/Protocols/VNC.py`, `RDP.py`, `WebRTC.py` (placeholders).
- API Endpoints:
- `POST /api/tunnel/request` → allocates lease, returns {tunnel_id, port, token, idle_seconds, grace_seconds, domain, protocol}.
- `DELETE /api/tunnel/<tunnel_id>` → operator-driven stop; pushes stop to agent and releases the lease.
- Domain default for PowerShell requests is `remote-interactive-shell` (legacy `ps` still accepted).
- Operator Socket.IO namespace `/tunnel`:
- `join`, `send`, `poll`, `ps_open`, `ps_send`, `ps_resize`, `ps_poll`.
- Operator socket disconnect triggers `stop_tunnel` if no other operators remain attached.
- WebUI (current): `Data/Engine/web-interface/src/Devices/ReverseTunnel/Powershell.jsx` requests PS leases in `remote-interactive-shell`, sends CLOSE frames, and calls DELETE on disconnect/unload.
## 3) Agent Components
- Role: `Data/Agent/Roles/role_ReverseTunnel.py`
- Validates signed lease tokens; enforces domain limits (2/1/2 with legacy fallbacks).
- Outbound TLS WS connect to assigned port; heartbeats + idle/grace watchdog; stop_all closes channels and sends CLOSE.
- Protocol registry: loads handlers from `Data/Agent/Roles/Reverse_Tunnels/*/Protocols/*` (PowerShell live; others stubbed to close unsupported channels cleanly).
- PowerShell channel: `Data/Agent/Roles/ReverseTunnel/tunnel_Powershell.py` (pipes-only, no PTY); re-exported under `Reverse_Tunnels/remote_interactive_shell/Protocols/Powershell.py`.
- Logging: `Agent/Logs/reverse_tunnel.log` with channel/tunnel lifecycle.
## 4) Framing, Heartbeats, Close
- Header: version(1) | msg_type(1) | flags(1) | reserved(1) | channel_id(u32 LE) | length(u32 LE).
- Messages: CONNECT/ACK, CHANNEL_OPEN/ACK, DATA, CONTROL (resize), WINDOW_UPDATE (reserved), HEARTBEAT (ping/pong), CLOSE.
- Close codes: ok, idle_timeout, grace_expired, protocol_error, auth_failed, server_shutdown, agent_shutdown, domain_limit, unexpected_disconnect.
- Heartbeats: Engine → Agent loop; idle/grace sweeper ~15s on Engine; Agent watchdog closes on idle/grace.
## 5) Lifecycle (PowerShell example)
1. UI calls `POST /api/tunnel/request` with agent_id, protocol=ps, domain=remote-interactive-shell.
2. Engine allocates port/tunnel_id, signs token, starts listener, pushes `reverse_tunnel_start` to agent.
3. Agent dials WS to assigned port, sends CONNECT with token. Engine validates, binds bridge, sends CONNECT_ACK + heartbeat.
4. Operator Socket.IO `/tunnel` joins; Engine attaches operator, instantiates PS server, issues CHANNEL_OPEN.
5. Agent launches PowerShell (pipes), streams stdout/stderr as DATA; operator input via `ps_send`; optional resize via `ps_resize` (no-op on agent pipes).
6. On operator Disconnect/tab close, UI sends CLOSE frame and calls DELETE; Engine stop path notifies agent (`reverse_tunnel_stop`), closes channel, releases lease/domain slot.
7. Idle/grace expiry or agent disconnect also triggers close/release; domain slots free immediately.
## 6) Security & Auth
- TLS: Reuse existing pinned bundle; outbound-only agent sockets.
- Token: short-lived, binds agent_id/tunnel_id/domain/protocol/port/expires_at; optional signature verification (Ed25519 signer when configured).
- Operator auth: uses existing Engine session/cookie/bearer for `/tunnel` namespace and API endpoints.
## 7) Configuration Knobs (defaults)
- Port pool: 3000040000; fixed port optional (context settings).
- Idle timeout: 3600s; Grace timeout: 3600s.
- Heartbeat interval: 20s (Engine → Agent).
- Domain limits: remote-interactive-shell=2, remote-management=1, remote-video=2; legacy aliases preserved.
- Log path: `Engine/Logs/reverse_tunnel.log`; `Agent/Logs/reverse_tunnel.log`.
## 8) Logs & Telemetry
- Engine: lease events, socket events, close reasons in `reverse_tunnel.log`; Device Activity start/stop with tunnel_id/operator_id when available.
- Agent: role lifecycle, channel start/stop, errors in `reverse_tunnel.log`.
## 9) Extending to New Protocols
- Add Engine handler under the appropriate domain folder and register in the orchestrators protocol registry.
- Add Agent handler under matching domain folder; update role registry to load it.
- Define channel open semantics (metadata), DATA/CONTROL usage, and close behavior.
- Update API/UI to allow selecting the protocol/domain and to send protocol-specific controls.
## 10) Outstanding Work
- Implement real handlers for Bash/SSH/WinRM/RDP/VNC/WebRTC and surface in UI.
- Add tests for DELETE stop path, per-domain limits, and browser disconnect cleanup.
- Consider a binary WebSocket browser bridge to replace Socket.IO for high-throughput protocols.
## 11) Risks & Watchpoints
- Eventlet/asyncio coexistence: tunnel loop runs on its own thread/loop; avoid blocking Socket.IO handlers.
- Port exhaustion: handle allocation failures cleanly; always release on stop/idle/grace.
- Buffer growth: add back-pressure before enabling high-throughput protocols.
- Security: strict token binding (agent_id/tunnel_id/domain/protocol/port/expiry) and TLS; reject framing errors.
## 12) Change Log (not exhaustive)
- 2025-11-30: Initial scaffold (lease manager, framing, tokens, API, Agent role, PS handlers).
- 2025-12-06: Simplified PS to pipes-only; improved handler imports; UI status tweaks.
- 2025-12-18: Domain lanes introduced (`remote-interactive-shell`, `remote-management`, `remote-video`) with limits 2/1/2; protocol handlers reorganized under `Reverse_Tunnels/*/Protocols/*`; orchestrator renamed to `reverse_tunnel_orchestrator.py`; explicit stop API/Socket.IO cleanup; WebUI Disconnect/unload calls DELETE + CLOSE for immediate teardown.