Merge pull request #147 from bunny-lab-io:codex/review-code-migration-tracker-instructions

Stage 5 - Bridge the legacy server to Engine APIs Implemented
This commit is contained in:
2025-10-26 03:51:43 -06:00
committed by GitHub
9 changed files with 336 additions and 79 deletions

View File

@@ -198,7 +198,7 @@ function Ensure-EngineWebInterface {
[string]$ProjectRoot
)
$engineSource = Join-Path $ProjectRoot 'Data\Engine\web-interface'
$engineSource = Join-Path $ProjectRoot 'Engine\web-interface'
$legacySource = Join-Path $ProjectRoot 'Data\Server\WebUI'
if (-not (Test-Path $legacySource)) {
@@ -1108,7 +1108,7 @@ switch ($choice) {
$venvFolder = "Server"
$dataSource = "Data"
$dataDestination = "$venvFolder\Borealis"
$customUIPath = "$dataSource\Engine\web-interface"
$customUIPath = (Join-Path $scriptDir 'Engine\web-interface')
$webUIDestination = "$venvFolder\web-interface"
$venvPython = Join-Path $venvFolder 'Scripts\python.exe'
@@ -1408,7 +1408,7 @@ switch ($choice) {
Run-Step "Copy Borealis Engine WebUI Files into: $webUIDestination" {
Ensure-EngineWebInterface -ProjectRoot $scriptDir
$engineWebUISource = Join-Path $engineSourceAbsolute 'web-interface'
$engineWebUISource = Join-Path $scriptDir 'Engine\web-interface'
if (Test-Path $engineWebUISource) {
$webUIDestinationAbsolute = Join-Path $scriptDir $webUIDestination
if (Test-Path $webUIDestinationAbsolute) {

View File

@@ -331,7 +331,7 @@ PY
}
ensure_engine_webui_source() {
local engineSource="Data/Engine/web-interface"
local engineSource="Engine/web-interface"
local legacySource="Data/Server/WebUI"
if [[ -d "${engineSource}/src" && -f "${engineSource}/package.json" ]]; then
return 0
@@ -351,7 +351,7 @@ ensure_engine_webui_source() {
}
prepare_webui() {
local customUIPath="Data/Engine/web-interface"
local customUIPath="Engine/web-interface"
local webUIDestination="Server/web-interface"
ensure_engine_webui_source || return 1
mkdir -p "$webUIDestination"

View File

@@ -27,10 +27,10 @@ Lastly, everytime that you complete a stage, you will create a pull request name
- [x] Provide fixtures that mirror the legacy SQLite schema and seed data.
- [x] Assert HTTP status codes, payloads, and side effects for parity.
- [x] Integrate Engine API tests into CI/local workflows.
- [ ] **Stage 5 — Bridge the legacy server to Engine APIs**
- [ ] Delegate API blueprint registration to the Engine factory from the legacy server.
- [ ] Replace legacy API routes with Engine-provided blueprints gated by a flag.
- [ ] Emit transitional logging when Engine handles requests.
- [x] **Stage 5 — Bridge the legacy server to Engine APIs**
- [x] Delegate API blueprint registration to the Engine factory from the legacy server.
- [x] Replace legacy API routes with Engine-provided blueprints gated by a flag.
- [x] Emit transitional logging when Engine handles requests.
- [ ] **Stage 6 — Plan WebUI migration**
- [ ] Move static/template handling into Data/Engine/services/WebUI.
- [ ] Preserve TLS-aware URL generation and caching.
@@ -43,5 +43,5 @@ Lastly, everytime that you complete a stage, you will create a pull request name
- [ ] Update legacy server to consume Engine WebSocket registration.
## Current Status
- **Stage:** Stage 4 — Build unit and smoke tests for Engine APIs (completed)
- **Stage:** Stage 5 — Bridge the legacy server to Engine APIs (completed)
- **Active Task:** Awaiting next stage instructions.

View File

@@ -112,8 +112,10 @@ def engine_harness(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[
key_path.write_text("test-key", encoding="utf-8")
bundle_path.write_text(bundle_contents, encoding="utf-8")
log_path = tmp_path / "logs" / "server.log"
log_path.parent.mkdir(parents=True, exist_ok=True)
logs_dir = tmp_path / "logs"
logs_dir.mkdir(parents=True, exist_ok=True)
log_path = logs_dir / "server.log"
error_log_path = logs_dir / "error.log"
config = {
"DATABASE_PATH": str(db_path),
@@ -121,6 +123,7 @@ def engine_harness(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[
"TLS_KEY_PATH": str(key_path),
"TLS_BUNDLE_PATH": str(bundle_path),
"LOG_FILE": str(log_path),
"ERROR_LOG_FILE": str(error_log_path),
"API_GROUPS": ("tokens", "enrollment"),
}

View File

@@ -8,16 +8,34 @@ legacy server defaults by binding to ``0.0.0.0:5001`` and honouring the
from __future__ import annotations
import logging
import os
import shutil
from pathlib import Path
from typing import Any, Dict
from .server import create_app
from .server import EngineContext, create_app
DEFAULT_HOST = "0.0.0.0"
DEFAULT_PORT = 5001
def _project_root() -> Path:
"""Locate the repository root by discovering the Borealis bootstrap script."""
current = Path(__file__).resolve().parent
for candidate in (current, *current.parents):
if (candidate / "Borealis.ps1").is_file():
return candidate
raise RuntimeError(
"Unable to locate the Borealis project root; Borealis.ps1 was not found "
"in any parent directory."
)
def _build_runtime_config() -> Dict[str, Any]:
return {
"HOST": os.environ.get("BOREALIS_ENGINE_HOST", DEFAULT_HOST),
@@ -28,16 +46,118 @@ def _build_runtime_config() -> Dict[str, Any]:
}
def _stage_web_interface_assets(logger: logging.Logger) -> None:
project_root = _project_root()
engine_web_root = project_root / "Engine" / "web-interface"
legacy_source = project_root / "Data" / "Server" / "WebUI"
if not legacy_source.is_dir():
raise RuntimeError(
f"Engine web interface source missing: {legacy_source}"
)
if engine_web_root.exists():
shutil.rmtree(engine_web_root)
shutil.copytree(legacy_source, engine_web_root)
index_path = engine_web_root / "index.html"
if not index_path.is_file():
raise RuntimeError(
f"Engine web interface staging failed; missing {index_path}"
)
logger.info(
"Engine web interface staged from %s to %s", legacy_source, engine_web_root
)
def _ensure_tls_material(context: EngineContext) -> None:
"""Ensure TLS certificate material exists, updating the context if created."""
try: # Lazy import so Engine still starts if legacy modules are unavailable.
from Modules.crypto import certificates # type: ignore
except Exception:
return
try:
cert_path, key_path, bundle_path = certificates.ensure_certificate()
except Exception as exc:
context.logger.error("Failed to auto-provision Engine TLS certificates: %s", exc)
return
cert_path_str = str(cert_path)
key_path_str = str(key_path)
bundle_path_str = str(bundle_path)
if not context.tls_cert_path or not Path(context.tls_cert_path).is_file():
context.tls_cert_path = cert_path_str
if not context.tls_key_path or not Path(context.tls_key_path).is_file():
context.tls_key_path = key_path_str
if not context.tls_bundle_path or not Path(context.tls_bundle_path).is_file():
context.tls_bundle_path = bundle_path_str
def _prepare_tls_run_kwargs(context: EngineContext) -> Dict[str, Any]:
"""Validate and return TLS arguments for the Socket.IO runner."""
_ensure_tls_material(context)
run_kwargs: Dict[str, Any] = {}
key_path_value = context.tls_key_path
if not key_path_value:
return run_kwargs
key_path = Path(key_path_value)
if not key_path.is_file():
raise RuntimeError(f"Engine TLS key file not found: {key_path}")
cert_candidates = []
if context.tls_bundle_path:
cert_candidates.append(context.tls_bundle_path)
if context.tls_cert_path and context.tls_cert_path not in cert_candidates:
cert_candidates.append(context.tls_cert_path)
if not cert_candidates:
raise RuntimeError("Engine TLS certificate path not configured; ensure certificates are provisioned.")
missing_candidates = []
for candidate in cert_candidates:
candidate_path = Path(candidate)
if candidate_path.is_file():
run_kwargs["certfile"] = str(candidate_path)
run_kwargs["keyfile"] = str(key_path)
return run_kwargs
missing_candidates.append(str(candidate_path))
checked = ", ".join(missing_candidates)
raise RuntimeError(f"Engine TLS certificate file not found. Checked: {checked}")
def main() -> None:
config = _build_runtime_config()
app, socketio, context = create_app(config)
try:
_stage_web_interface_assets(context.logger)
except Exception as exc:
context.logger.error("Failed to stage Engine web interface: %s", exc)
raise
host = config.get("HOST", DEFAULT_HOST)
port = int(config.get("PORT", DEFAULT_PORT))
run_kwargs: Dict[str, Any] = {"host": host, "port": port}
if context.tls_bundle_path and context.tls_key_path:
run_kwargs.update({"certfile": context.tls_bundle_path, "keyfile": context.tls_key_path})
try:
tls_kwargs = _prepare_tls_run_kwargs(context)
except RuntimeError as exc:
context.logger.error("TLS configuration error: %s", exc)
raise
else:
if tls_kwargs:
run_kwargs.update(tls_kwargs)
context.logger.info("Engine TLS enabled using certificate %s", tls_kwargs["certfile"])
socketio.run(app, **run_kwargs)

View File

@@ -26,8 +26,9 @@ defaults that mirror the legacy server runtime. Key environment variables are
When TLS values are not provided explicitly the Engine falls back to the
certificate helper shipped with the legacy server, ensuring bundling parity.
Logs are written to ``Logs/Server/server.log`` with daily rotation so the new
runtime integrates with existing operational practices.
Logs are written to ``Logs/Engine/engine.log`` with daily rotation and
errors are additionally duplicated to ``Logs/Engine/error.log`` so the
runtime integrates with the platform's logging policy.
"""
from __future__ import annotations
@@ -48,7 +49,9 @@ except Exception: # pragma: no-cover - Engine configuration still works without
ENGINE_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = ENGINE_DIR.parent.parent
DEFAULT_DATABASE_PATH = PROJECT_ROOT / "database.db"
LOG_FILE_PATH = PROJECT_ROOT / "Logs" / "Server" / "server.log"
LOG_ROOT = PROJECT_ROOT / "Logs" / "Engine"
LOG_FILE_PATH = LOG_ROOT / "engine.log"
ERROR_LOG_FILE_PATH = LOG_ROOT / "error.log"
def _ensure_parent(path: Path) -> None:
@@ -61,16 +64,28 @@ def _ensure_parent(path: Path) -> None:
def _resolve_static_folder() -> str:
candidates = [
ENGINE_DIR / "web-interface" / "build",
ENGINE_DIR / "web-interface" / "dist",
candidate_roots = [
PROJECT_ROOT / "Engine" / "web-interface",
ENGINE_DIR / "web-interface",
PROJECT_ROOT / "Data" / "Server" / "web-interface",
]
candidates = []
for root in candidate_roots:
absolute_root = root.resolve()
candidates.extend(
[
absolute_root / "build",
absolute_root / "dist",
absolute_root,
]
)
for candidate in candidates:
absolute = candidate.resolve()
if absolute.is_dir():
return str(absolute)
return str(candidates[0].resolve())
if candidate.is_dir():
return str(candidate)
return str(candidates[0])
def _parse_origins(raw: Optional[Any]) -> Optional[List[str]]:
@@ -139,6 +154,7 @@ class EngineSettings:
tls_key_path: Optional[str]
tls_bundle_path: Optional[str]
log_file: str
error_log_file: str
api_groups: Tuple[str, ...]
raw: MutableMapping[str, Any] = field(default_factory=dict)
@@ -219,6 +235,9 @@ def load_runtime_config(overrides: Optional[Mapping[str, Any]] = None) -> Engine
log_file = str(runtime_config.get("LOG_FILE") or LOG_FILE_PATH)
_ensure_parent(Path(log_file))
error_log_file = str(runtime_config.get("ERROR_LOG_FILE") or ERROR_LOG_FILE_PATH)
_ensure_parent(Path(error_log_file))
api_groups = _parse_api_groups(
runtime_config.get("API_GROUPS") or os.environ.get("BOREALIS_API_GROUPS")
)
@@ -237,6 +256,7 @@ def load_runtime_config(overrides: Optional[Mapping[str, Any]] = None) -> Engine
tls_key_path=tls_key_path if tls_key_path else None,
tls_bundle_path=tls_bundle_path if tls_bundle_path else None,
log_file=str(log_file),
error_log_file=str(error_log_file),
api_groups=api_groups,
raw=runtime_config,
)
@@ -244,7 +264,7 @@ def load_runtime_config(overrides: Optional[Mapping[str, Any]] = None) -> Engine
def initialise_engine_logger(settings: EngineSettings, name: str = "borealis.engine") -> logging.Logger:
"""Configure the Engine logger to write to the shared server log."""
"""Configure the Engine logger to write to Engine log files."""
logger = logging.getLogger(name)
if not logger.handlers:
@@ -263,6 +283,16 @@ def initialise_engine_logger(settings: EngineSettings, name: str = "borealis.eng
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
error_handler = TimedRotatingFileHandler(
settings.error_log_file,
when="midnight",
backupCount=0,
encoding="utf-8",
)
error_handler.setLevel(logging.ERROR)
error_handler.setFormatter(formatter)
logger.addHandler(error_handler)
logger.setLevel(logging.INFO)
logger.propagate = False
return logger

View File

@@ -0,0 +1,9 @@
#////////// PROJECT FILE SEPARATION LINE ////////// CODE AFTER THIS LINE ARE FROM: <ProjectRoot>/Data/Engine/engine-requirements.txt
Flask
flask-cors
flask_socketio
eventlet
cryptography
PyJWT[crypto]
pyotp
qrcode

View File

@@ -4,11 +4,12 @@ Stage 1 introduced the structural skeleton for the Engine runtime. Stage 2
builds upon that foundation by centralising configuration handling and logging
initialisation so the Engine mirrors the legacy server's start-up behaviour.
The factory delegates configuration resolution to :mod:`Data.Engine.config`
and emits structured logs to ``Logs/Server/server.log`` to align with the
project's operational practices.
and emits structured logs to ``Logs/Engine/engine.log`` (with an accompanying
error log) to align with the project's operational practices.
"""
from __future__ import annotations
import importlib.util
import logging
import ssl
import sys
@@ -16,22 +17,31 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Any, Mapping, Optional, Sequence, Tuple
import eventlet
from flask import Flask
from flask_cors import CORS
from flask_socketio import SocketIO
from werkzeug.middleware.proxy_fix import ProxyFix
# Eventlet ensures Socket.IO long-polling and WebSocket support parity with the
# legacy server. We keep thread pools enabled for compatibility with blocking
# filesystem/database operations.
eventlet.monkey_patch(thread=False)
def _require_dependency(module: str, friendly_name: str) -> None:
if importlib.util.find_spec(module) is None: # pragma: no cover - import check
raise RuntimeError(
f"{friendly_name} (Python module '{module}') is required for the Borealis Engine runtime. "
"Install the packaged dependencies by running Borealis.ps1 or ensure the module is present in the active environment."
)
try: # pragma: no-cover - defensive import mirroring the legacy runtime.
from eventlet.wsgi import HttpProtocol # type: ignore
except Exception: # pragma: no-cover - the Engine should still operate without it.
HttpProtocol = None # type: ignore[assignment]
else:
_require_dependency("eventlet", "Eventlet")
_require_dependency("flask", "Flask")
_require_dependency("flask_socketio", "Flask-SocketIO")
import eventlet # type: ignore # noqa: E402 # pragma: no cover - import guarded above
from eventlet import wsgi as eventlet_wsgi # type: ignore # noqa: E402 # pragma: no cover
from flask import Flask, request # noqa: E402
from flask_cors import CORS # noqa: E402
from flask_socketio import SocketIO # noqa: E402
from werkzeug.middleware.proxy_fix import ProxyFix # noqa: E402
eventlet.monkey_patch(thread=False) # pragma: no cover - aligns with legacy runtime
HttpProtocol = getattr(eventlet_wsgi, "HttpProtocol", None)
if HttpProtocol is not None: # pragma: no branch - attribute exists in supported versions
_original_handle_one_request = HttpProtocol.handle_one_request
def _quiet_tls_http_mismatch(self): # type: ignore[override]
@@ -74,6 +84,8 @@ else:
HttpProtocol.handle_one_request = _quiet_tls_http_mismatch # type: ignore[assignment]
_SOCKETIO_ASYNC_MODE = "eventlet"
# Ensure the legacy ``Modules`` package is importable when running from the
# Engine deployment directory.
@@ -81,6 +93,7 @@ _ENGINE_DIR = Path(__file__).resolve().parent
_SEARCH_ROOTS = [
_ENGINE_DIR.parent / "Server",
_ENGINE_DIR.parent.parent / "Data" / "Server",
_ENGINE_DIR.parent.parent.parent / "Data" / "Server",
]
for root in _SEARCH_ROOTS:
modules_dir = root / "Modules"
@@ -106,7 +119,46 @@ class EngineContext:
api_groups: Sequence[str]
__all__ = ["EngineContext", "create_app"]
__all__ = ["EngineContext", "create_app", "register_engine_api"]
def _build_engine_context(settings: EngineSettings, logger: logging.Logger) -> EngineContext:
return EngineContext(
database_path=settings.database_path,
logger=logger,
scheduler=None,
tls_cert_path=settings.tls_cert_path,
tls_key_path=settings.tls_key_path,
tls_bundle_path=settings.tls_bundle_path,
config=settings.as_dict(),
api_groups=settings.api_groups,
)
def _attach_transition_logging(app: Flask, context: EngineContext, logger: logging.Logger) -> None:
tracked = {group.strip().lower() for group in context.api_groups if group}
if not tracked:
tracked = {"tokens", "enrollment"}
existing = getattr(app, "_engine_api_tracked_blueprints", set())
if existing:
tracked.update(existing)
setattr(app, "_engine_api_tracked_blueprints", tracked)
if getattr(app, "_engine_api_logging_installed", False):
return
@app.before_request
def _log_engine_api_bridge() -> None: # pragma: no cover - integration behaviour exercised in higher-level tests
blueprint = (request.blueprint or "").lower()
if blueprint and blueprint in getattr(app, "_engine_api_tracked_blueprints", tracked):
logger.info(
"Engine handling API request via legacy bridge: %s %s",
request.method,
request.path,
)
setattr(app, "_engine_api_logging_installed", True)
def create_app(config: Optional[Mapping[str, Any]] = None) -> Tuple[Flask, SocketIO, EngineContext]:
@@ -115,8 +167,6 @@ def create_app(config: Optional[Mapping[str, Any]] = None) -> Tuple[Flask, Socke
settings: EngineSettings = load_runtime_config(config)
logger = initialise_engine_logger(settings)
database_path = settings.database_path
static_folder = settings.static_folder
app = Flask(__name__, static_folder=static_folder, static_url_path="")
app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)
@@ -134,29 +184,14 @@ def create_app(config: Optional[Mapping[str, Any]] = None) -> Tuple[Flask, Socke
socketio = SocketIO(
app,
cors_allowed_origins="*",
async_mode="eventlet",
async_mode=_SOCKETIO_ASYNC_MODE,
engineio_options={
"max_http_buffer_size": 100_000_000,
"max_websocket_message_size": 100_000_000,
},
)
tls_cert_path, tls_key_path, tls_bundle_path = (
settings.tls_cert_path,
settings.tls_key_path,
settings.tls_bundle_path,
)
context = EngineContext(
database_path=database_path,
logger=logger,
scheduler=None,
tls_cert_path=tls_cert_path,
tls_key_path=tls_key_path,
tls_bundle_path=tls_bundle_path,
config=settings.as_dict(),
api_groups=settings.api_groups,
)
context = _build_engine_context(settings, logger)
from .services import API, WebSocket, WebUI # Local import to avoid circular deps during bootstrap
@@ -167,3 +202,21 @@ def create_app(config: Optional[Mapping[str, Any]] = None) -> Tuple[Flask, Socke
logger.debug("Engine application factory completed initialisation.")
return app, socketio, context
def register_engine_api(app: Flask, *, config: Optional[Mapping[str, Any]] = None) -> EngineContext:
"""Register Engine-managed API blueprints onto an existing Flask app."""
settings: EngineSettings = load_runtime_config(config)
logger = initialise_engine_logger(settings)
context = _build_engine_context(settings, logger)
from .services import API # Local import avoids circular dependency at module import time
API.register_api(app, context)
_attach_transition_logging(app, context, logger)
groups_display = ", ".join(context.api_groups) if context.api_groups else "none"
logger.info("Engine API delegation activated for groups: %s", groups_display)
return context

View File

@@ -210,6 +210,13 @@ def _infer_server_scope(message: str, explicit: Optional[str]) -> Optional[str]:
return None
def _env_flag(name: str, *, default: bool = False) -> bool:
raw = os.environ.get(name)
if raw is None:
return default
return raw.strip().lower() in {"1", "true", "yes", "on"}
def _is_internal_request(req: Request) -> bool:
"""Return True if the HTTP request originated from the local server host."""
try:
@@ -326,6 +333,8 @@ AUTH_RATE_LIMITER = SlidingWindowRateLimiter()
ENROLLMENT_NONCE_CACHE = NonceCache()
DPOP_VALIDATOR = DPoPValidator()
DEVICE_AUTH_MANAGER: Optional[DeviceAuthManager] = None
ENGINE_API_ENABLED = _env_flag("BOREALIS_ENGINE_API")
ENGINE_API_GROUPS: Tuple[str, ...] = tuple()
def _set_cached_github_token(token: Optional[str]) -> None:
@@ -5088,24 +5097,57 @@ def init_db():
init_db()
enrollment_routes.register(
app,
db_conn_factory=_db_conn,
log=_write_service_log,
jwt_service=JWT_SERVICE,
tls_bundle_path=TLS_BUNDLE_PATH,
ip_rate_limiter=IP_RATE_LIMITER,
fp_rate_limiter=FP_RATE_LIMITER,
nonce_cache=ENROLLMENT_NONCE_CACHE,
script_signer=SCRIPT_SIGNER,
)
if ENGINE_API_ENABLED:
_engine_api_config: Dict[str, Any] = {
"DATABASE_PATH": DB_PATH,
"TLS_CERT_PATH": TLS_CERT_PATH,
"TLS_KEY_PATH": TLS_KEY_PATH,
"TLS_BUNDLE_PATH": TLS_BUNDLE_PATH,
}
api_groups_override = os.environ.get("BOREALIS_API_GROUPS")
if api_groups_override:
_engine_api_config["API_GROUPS"] = api_groups_override
token_routes.register(
app,
db_conn_factory=_db_conn,
jwt_service=JWT_SERVICE,
dpop_validator=DPOP_VALIDATOR,
)
try:
from Data.Engine.server import register_engine_api
_engine_context = register_engine_api(app, config=_engine_api_config)
except Exception:
ENGINE_API_ENABLED = False
ENGINE_API_GROUPS = tuple()
_write_service_log(
"server",
"Engine API delegation failed; continuing with legacy API registration.",
level="ERROR",
)
else:
ENGINE_API_GROUPS = tuple(_engine_context.api_groups)
_write_service_log(
"server",
"Engine API delegation enabled for groups: {}".format(
", ".join(ENGINE_API_GROUPS) or "default"
),
)
if not ENGINE_API_ENABLED:
enrollment_routes.register(
app,
db_conn_factory=_db_conn,
log=_write_service_log,
jwt_service=JWT_SERVICE,
tls_bundle_path=TLS_BUNDLE_PATH,
ip_rate_limiter=IP_RATE_LIMITER,
fp_rate_limiter=FP_RATE_LIMITER,
nonce_cache=ENROLLMENT_NONCE_CACHE,
script_signer=SCRIPT_SIGNER,
)
token_routes.register(
app,
db_conn_factory=_db_conn,
jwt_service=JWT_SERVICE,
dpop_validator=DPOP_VALIDATOR,
)
agent_routes.register(
app,